<commit_before>test(
  'CharsTest',

  [
    'ephox.compass.Arr',
    'ephox.polaris.pattern.Chars',
    'global!RegExp'
  ],

  function (Arr, Chars, RegExp) {
// ["fr","en_au","pt","it","nl","en_uk","pt_pt","de","nb","en_br","sv","da","en","es","en_gb","fi","en_us"]}

    /*
     * https://www.cs.tut.fi/~jkorpela/html/french.html
     */
    var regex = new RegExp(Chars.wordchar(), '');
    var french = 'àÀâÂèÈéÉêÊëËîÎïÏôÔùÙûÛüÜÿŸçÇœŒ';

    var checkAllKnown = function (label, str) {
      var chars = str.split('');
      var breaks = Arr.filter(chars, function (c) {
        return !regex.test(c);
      });

      var leftovers = breaks.join('').trim();
      assert.eq(
        0,
        leftovers.length,
        'Test: ' + label + '\nExpected all characters in: \n\n"' + str + '" to be known. \nUnknown: ' + leftovers
      );
    };

    checkAllKnown('Checking French string', 'àÀâÂèÈéÉêÊëËîÎïÏôÔùÙûÛüÜÿŸçÇœŒ');
  }
);<commit_msg>TBIO-4391: checking characters<commit_after>test(
  'CharsTest',

  [
    'ephox.compass.Arr',
    'ephox.polaris.pattern.Chars',
    'global!RegExp'
  ],

  function (Arr, Chars, RegExp) {
// ["fr","en_au","pt","it","nl","en_uk","pt_pt","de","nb","en_br","sv","da","en","es","en_gb","fi","en_us"]}

    var supported = ['fr','en_au','pt','it','nl','en_uk','pt_pt','de','nb','en_br','sv','da','en','es','en_gb','fi','en_us'];


    var extras = {
      fr: {
        chars: 'àÀâÂèÈéÉêÊëËîÎïÏôÔùÙûÛüÜÿŸçÇœŒ',
        html: 'https://www.cs.tut.fi/~jkorpela/html/french.html',
        label: 'French language',
        code: 'fr'
      },
      en_au: { },
      pt: {

      },
      it: {

      },
      nl: {

      },
      en_uk: {

      },
      pt_pt: {

      },
      de: {

      },
      nb: {

      },
      en_br: { },
      sv: {

      },
      da: {

      },
      en: { },
      es: {

      },
      en_gb: {

      },
      fi: {

      },
      en_us: { }
    };

    /*
     * https://www.cs.tut.fi/~jkorpela/html/french.html
     */
    var regex = new RegExp(Chars.wordchar(), '');

    var checkAllKnown = function (label, str) {
      var chars = str.split('');
      var breaks = Arr.filter(chars, function (c) {
        return !regex.test(c);
      });

      var leftovers = breaks.join('').trim();
      assert.eq(
        0,
        leftovers.length,
        'Test: ' + label + '\nExpected all characters in: \n\n"' + str + '" to be known. \nUnknown: ' + leftovers
      );
    };

    Arr.each(supported, function (code) {
      var info = extras[code];
      checkAllKnown(info.label, info.chars);
    });
  }
);
{'fraction_non_alphanumeric': 0.20987654320987653, 'fraction_numerical': 0.0023148148148148147, 'mean_word_length': 2.1278648974668277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2543748', 'n_tokens_mistral': 1057, 'n_tokens_neox': 1028, 'n_words': 146}