<commit_before>var path = require('path');
var _ = require('lodash');
var objParser = require('l10n-tools/object-extractor');
var extendWhile = require('../utils/extend-while');

module.exports = function(assemble) {
  //var mergeTranslated = require('../utils/merge-tranlated-dictionary');
  var parseFilePath = require('../utils/parse-file-path')(assemble);
  var websiteRoot = assemble.get('data.websiteRoot');
  var locales = assemble.get('data.locales');
  var removeTranslationKeys = require('../utils/remove-translation-keys');
  var isTest = assemble.get('env') === 'test';
  var lastLocale;

  return function mergeTranslatedData (file, next) {
    var lang = assemble.get('lang');
    var subfoldersRoot = assemble.get('data.subfoldersRoot');
    var rootData = assemble.get('rootData');
    var translated = assemble.get('translated');
    var dicts = assemble.get('dicts');
    var filePathData = parseFilePath(file.path);
    var locale = isTest ? 'de' : filePathData.locale;
    var dataKey = filePathData.dataKey;
    var dictKey = locales[locale];
    var mergedDict, parentKey, translatedDict, fileDataClone, layoutKeys;

    removeTranslationKeys(file.data);

    if(file.data.isPpc) {
      return next();
    }

    //extend the file with the external YML content
    if(filePathData.isRoot && !isTest && !file.data.isPpc) {
      //extend the local yml data to the page
      fileDataClone = rootData[dataKey];

      if(fileDataClone) {

        //TODO: figure out how to do this transform earlier
        if(fileDataClone.page_content) {
          file.content = fileDataClone.page_content;
        }

        extendWhile(file.data, fileDataClone);
      }

    } else if(filePathData.isSubfolder || ( filePathData.isRoot && isTest && !file.data.isPpc )) {
      if(isTest) {
        dataKey = dataKey.replace('/' + websiteRoot + '/', '/' + path.join(subfoldersRoot, locale) + '/');
      }

      //set the locale on the page context for modal|partial translation
      file.data.locale = locale;
      file.data.dataKey = dataKey;
      translatedDict = translated[dictKey] && translated[dictKey][dataKey];

      if(translatedDict) {

        //TODO: figure out how to do this transform earlier
        if(translatedDict.page_content) {
          file.content = translatedDict.page_content;
        }

        extendWhile(file.data, translatedDict);
      }

    } else if ( (isTest || ( file.data.locale && file.data.locale !== websiteRoot ) ) && ( filePathData.isModal || filePathData.isPartial ) ) {
      locale = file.data.locale;
      dictKey = locales[locale];
      file.data.dataKey = dataKey;

      translatedDict = translated[dictKey] && translated[dictKey][dataKey];

      //TODO: for now modals/partials are not locale specific, in future may have locale specific
      //partials that possible overwrite parent partial data
      if(!file.data.isPpc && translatedDict) {
        extendWhile(file.data, translatedDict);
      }

    }

    //deal with global data
    //this assumes that modals and partials don't access global data
    if(locale !== lastLocale && !file.data.isPpc && !filePathData.isModal && !filePathData.isPartial) {
      var globalData = assemble.get('data');

      if(isTest || filePathData.isSubfolder) {
        Object.keys(globalData).forEach(function(key) {
          if(/global\_/.test(key)) {
            //intentionally mutate assemble.cache.data
            globalData[key] = translated[dictKey].global[key];
          }
        });
      }<commit_msg>translate special types<commit_after>var path = require('path');
var _ = require('lodash');
var objParser = require('l10n-tools/object-extractor');
var extendWhile = require('../utils/extend-while');

module.exports = function(assemble) {
  //var mergeTranslated = require('../utils/merge-tranlated-dictionary');
  var parseFilePath = require('../utils/parse-file-path')(assemble);
  var websiteRoot = assemble.get('data.websiteRoot');
  var locales = assemble.get('data.locales');
  var removeTranslationKeys = require('../utils/remove-translation-keys');
  var isTest = assemble.get('env') === 'test';
  var lastLocale;

  var lastLength, log = true;
  return function mergeTranslatedData (file, next) {
    var lang = assemble.get('lang');
    var subfoldersRoot = assemble.get('data.subfoldersRoot');
    var rootData = assemble.get('rootData');
    var translated = assemble.get('translated');
    var dicts = assemble.get('dicts');
    var filePathData = parseFilePath(file.path);
    var locale = isTest ? 'de' : filePathData.locale;
    var dataKey = filePathData.dataKey;
    var dictKey = locales[locale];
    var mergedDict, parentKey, translatedDict, fileDataClone, layoutKeys;

    removeTranslationKeys(file.data);

    if(file.data.isPpc) {
      return next();
    }

    //extend the file with the external YML content
    if(filePathData.isRoot && !isTest && !file.data.isPpc) {
      //extend the local yml data to the page
      fileDataClone = rootData[dataKey];

      if(fileDataClone) {

        //TODO: figure out how to do this transform earlier
        if(fileDataClone.page_content) {
          file.content = fileDataClone.page_content;
        }

        extendWhile(file.data, fileDataClone);
      }

    } else if(filePathData.isSubfolder || ( filePathData.isRoot && isTest && !file.data.isPpc )) {
      if(isTest) {
        dataKey = dataKey.replace('/' + websiteRoot + '/', '/' + path.join(subfoldersRoot, locale) + '/');
      }

      //set the locale on the page context for modal|partial translation
      file.data.locale = locale;
      file.data.dataKey = dataKey;
      translatedDict = translated[dictKey] && translated[dictKey][dataKey];

      if(translatedDict) {

        //TODO: figure out how to do this transform earlier
        if(translatedDict.page_content) {
          file.content = translatedDict.page_content;
        }

        extendWhile(file.data, translatedDict);
      }

    } else if ( (isTest || ( file.data.locale && file.data.locale !== websiteRoot ) ) && ( filePathData.isModal || filePathData.isPartial ) ) {
      locale = file.data.locale;
      dictKey = locales[locale];
      file.data.dataKey = dataKey;

      translatedDict = translated[dictKey] && translated[dictKey][dataKey];

      //TODO: for now modals/partials are not locale specific, in future may have locale specific
      //partials that possible overwrite parent partial data
      if(!file.data.isPpc && translatedDict) {
        //TODO: helper_phrases were getting mutated for signin_modal and many more properties were being added to arrray
        //unsure of the source of this mutation
        file.data = _.merge({}, file.data, translatedDict);
      }

    }

    //deal with global data
    //this assumes that modals and partials don't access global data
    if(locale !== lastLocale && !file.data.isPpc && !filePathData.isModal && !filePathData.isPartial) {
      var globalData = assemble.get('data');

      if(isTest || filePathData.isSubfolder) {
        Object.keys(globalData).forEach(function(key) {
          if(/global\_/.test(key)) {
            //intentionally mutate assemble.cache.data
            globalData[key] = translated[dictKey].global[key];
          }
        });
      }
{'fraction_non_alphanumeric': 0.12814519455520418, 'fraction_numerical': 0.0005499793757734085, 'mean_word_length': 3.0366259711431742, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4663525', 'n_tokens_mistral': 2174, 'n_tokens_neox': 2100, 'n_words': 491}