<commit_before>module.exports = function(grunt, options) {

  'use strict';

  var fs = require('fs');
  var path = require('path');
  var util = require('util');

  var langMetaData = require('../data/languages-metadata');
  var enabledLanguages = {};
  var enabledLabels = {};

  var suffixRegExp = /\/Localizable\.strings$/;
  var lineParsingRegExp = /^\s*\"([a-zA-Z0-9_\-\$]+)\"\s*=\s*\"(.*)\";\s*$/;
  var template = "define(function() {\nreturn {\n'name': '%s',\n'data': %s\n};\n});";

  function resolveLangCode (langCode) {

    // GetLocalization doesn't follow ISO codes, fix the names
    langCode = langCode.replace(/\-/g, '_');

    var metaData = langMetaData[langCode];
    while (metaData && metaData.alias) {
      langCode = metaData.alias;
      metaData = langMetaData[langCode];
    }

    // if there is no metadata for this language, then drop out
    if(!metaData) {
      grunt.warn('Language missing:' + langCode);
      return;
    }

    return {
      'code': langCode,
      'data': metaData
    };
  }

  function fileToCode (file, options) {
    var prefixRegexp = new RegExp('^' + options.src + '/');
    return file.replace(prefixRegexp, '').replace(suffixRegExp, '');
  }

  function name (file, options) {
    var langCode = fileToCode(file, options);
    var resolved = resolveLangCode(langCode);
    return resolved && resolved.data.file;
  }

  function compile (rawLanguageData, options, callback) {

    var json = {};

    var lines = rawLanguageData.split(/[\r\n]+/);
    lines.forEach(function(line) {

      var sections = line.match(lineParsingRegExp);
      if (sections && sections.length >= 2 && !sections[1].match(/\s/)) {

        var key = sections[1];
        if (!(enabledLabels[key])) { // TODO: how to handle momentjs ?? && !key.match(/^momentjs_/)) {
          return;
        }

        var value = sections[2];
        value = value.replace(/(%|@)(([0-9]+)\$)?(@|%)/g, function(x, y, num) {
          return '$' + (num || '');
        }).replace(/\\\"/g, '"');

        json[key] = value;
      }
    });

    callback(null, JSON.stringify(json));
  }

  function compileAvailable(err, languages, options, done) {
<commit_msg>handle rtl language strings better<commit_after>module.exports = function(grunt, options) {

  'use strict';

  var fs = require('fs');
  var path = require('path');
  var util = require('util');

  var langMetaData = require('../data/languages-metadata');
  var enabledLanguages = {};
  var enabledLabels = {};

  var suffixRegExp = /\/Localizable\.strings$/;
  var lineParsingRegExp = /^\s*\"([a-zA-Z0-9_\-\$]+)\"\s*=\s*\"(.*)\";\s*$/;
  var template = "define(function() {\nreturn {\n'name': '%s',\n'data': %s\n};\n});";

  function resolveLangCode (langCode) {

    // GetLocalization doesn't follow ISO codes, fix the names
    langCode = langCode.replace(/\-/g, '_');

    var metaData = langMetaData[langCode];
    while (metaData && metaData.alias) {
      langCode = metaData.alias;
      metaData = langMetaData[langCode];
    }

    // if there is no metadata for this language, then drop out
    if(!metaData) {
      grunt.warn('Language missing:' + langCode);
      return;
    }

    return {
      'code': langCode,
      'data': metaData
    };
  }

  function fileToCode (file, options) {
    var prefixRegexp = new RegExp('^' + options.src + '/');
    return file.replace(prefixRegexp, '').replace(suffixRegExp, '');
  }

  function name (file, options) {
    var langCode = fileToCode(file, options);
    var resolved = resolveLangCode(langCode);
    return resolved && resolved.data.file;
  }

  function replaceWithNum (x, y, num) {
    return '$' + (num || '');
  }

  function compile (rawLanguageData, options, callback) {

    var json = {};

    var lines = rawLanguageData.split(/[\r\n]+/);
    lines.forEach(function(line) {

      var sections = line.match(lineParsingRegExp);
      if (sections && sections.length >= 2 && !sections[1].match(/\s/)) {

        var key = sections[1];
        if (!(enabledLabels[key])) { // TODO: how to handle momentjs ?? && !key.match(/^momentjs_/)) {
          return;
        }

        var value = sections[2];

        var regexps = [
          /%((\d+)\$)?@/g,
          /@((\d+)\$)?%/g
        ];

        regexps.forEach(function (regexp) {
          value = value.replace(regexp, replaceWithNum);
        });

        value = value.replace(/\\\"/g, '"');

        json[key] = value;
      }
    });

    callback(null, JSON.stringify(json));
  }

  function compileAvailable(err, languages, options, done) {

{'fraction_non_alphanumeric': 0.18269441401971523, 'fraction_numerical': 0.0030668127053669223, 'mean_word_length': 2.697165991902834, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '761521', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1449, 'n_words': 324}