<commit_before>import { Logger } from '@ephox/agar';
import { Assert, UnitTest } from '@ephox/bedrock-client';
import { Gene, TestUniverse, TextGene } from '@ephox/boss';
import { Arr, Option } from '@ephox/katamari';
import Jsc from '@ephox/wrap-jsverify';
import { ArbTextIds, arbTextIds } from 'ephox/robin/test/Arbitraries';
import Clustering from 'ephox/robin/words/Clustering';
import { WordDecisionItem } from 'ephox/robin/words/WordDecision';
import { LanguageZones } from 'ephox/robin/zone/LanguageZones';

UnitTest.test('ClusteringTest', function () {
  const checkWords = function (universe: TestUniverse, words: WordDecisionItem<Gene>[]) {
    return Arr.map(words, function (a) {
      const text = universe.property().getText(a.item());
      return text.substring(a.start(), a.finish());
    });
  };

  const check = function (label: string, universe: TestUniverse, expLeft: string[], expMiddle: string[], expRight: string[], expLang: Option<string>, id: string) {
    Logger.sync(
      id + ' => check: ' + label,
      function () {
        const act = Clustering.byLanguage(universe, universe.find(universe.get(), id).getOrDie());
        Assert.eq('start: ' + id + ', check left()', expLeft, checkWords(universe, act.left()));
        Assert.eq('start: ' + id + ', check middle()', expMiddle, checkWords(universe, act.middle()));
        Assert.eq('start: ' + id + ', check right()', expRight, checkWords(universe, act.right()));
        Assert.eq(
          () => 'start: ' + id + ', check lang(): expected: ' + expLang.toString() + ', actual: ' + act.lang().toString(),
          true, expLang.equals(act.lang())
        );
        // .all() is:  tfel + middle + right
        Assert.eq('start: ' + id + ', check all()', expLeft.reverse().concat(expMiddle).concat(expRight), checkWords(universe, act.all()));
      }
    );
  };

  const testSanity = function () {<commit_msg>Use Arr.reverse instead of Array.prototype.reverse to avoid mutation in expression.<commit_after>import { Logger } from '@ephox/agar';
import { Assert, UnitTest } from '@ephox/bedrock-client';
import { Gene, TestUniverse, TextGene } from '@ephox/boss';
import { Arr, Option } from '@ephox/katamari';
import Jsc from '@ephox/wrap-jsverify';
import { ArbTextIds, arbTextIds } from 'ephox/robin/test/Arbitraries';
import Clustering from 'ephox/robin/words/Clustering';
import { WordDecisionItem } from 'ephox/robin/words/WordDecision';
import { LanguageZones } from 'ephox/robin/zone/LanguageZones';

UnitTest.test('ClusteringTest', function () {
  const checkWords = function (universe: TestUniverse, words: WordDecisionItem<Gene>[]) {
    return Arr.map(words, function (a) {
      const text = universe.property().getText(a.item());
      return text.substring(a.start(), a.finish());
    });
  };

  const check = function (label: string, universe: TestUniverse, expLeft: string[], expMiddle: string[], expRight: string[], expLang: Option<string>, id: string) {
    Logger.sync(
      id + ' => check: ' + label,
      function () {
        const act = Clustering.byLanguage(universe, universe.find(universe.get(), id).getOrDie());
        Assert.eq('start: ' + id + ', check left()', expLeft, checkWords(universe, act.left()));
        Assert.eq('start: ' + id + ', check middle()', expMiddle, checkWords(universe, act.middle()));
        Assert.eq('start: ' + id + ', check right()', expRight, checkWords(universe, act.right()));
        Assert.eq(
          () => 'start: ' + id + ', check lang(): expected: ' + expLang.toString() + ', actual: ' + act.lang().toString(),
          true, expLang.equals(act.lang())
        );
        // .all() is:  tfel + middle + right
        Assert.eq('start: ' + id + ', check all()', Arr.reverse(expLeft).concat(expMiddle).concat(expRight), checkWords(universe, act.all()));
      }
    );
  };

  const testSanity = function () {
{'fraction_non_alphanumeric': 0.17514854042882977, 'fraction_numerical': 0.0, 'mean_word_length': 3.768472906403941, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4040443', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1215, 'n_words': 277}