<commit_before>define(
  'ephox.robin.words.Clustering',

  [
    'ephox.bud.Unicode',
    'ephox.compass.Arr',
    'ephox.peanut.Fun',
    'ephox.phoenix.api.general.Extract',
    'ephox.phoenix.api.general.Gather',
    'ephox.robin.words.WordDecision',
    'ephox.robin.words.WordWalking'
  ],

  function (Unicode, Arr, Fun, Extract, Gather, WordDecision, WordWalking) {
    /*
     * Identification of words:
     *
     * For boundaries, stop the gathering process and do not include
     * For empty tags, stop the gathering process and do not include
     * For text nodes:
     *   a) text node has a character break, stop the gathering process and include partial
     *   b) text node has no character breaks, keep gathering and include entire node
     * For other elements, calculate the language of the closest ancestor:
     *   a) if the (destination) language has changed, stop the gathering process and do not incude
     *   b) if the (destination) language has not changed, keep gathering and do not include
     * These rules are encoded in WordDecision.decide
     * Returns: [WordDecision.make Struct] of all the words recursively from item in direction.
     */
    var doWords = function (universe, item, mode, direction, currLanguage) {
      var destination = Gather.walk(universe, item, mode, direction);
      var result = destination.map(function (dest) {
        var decision = WordDecision.decide(universe, dest.item(), direction.slicer, currLanguage);
        var recursive = decision.abort() ? [] : doWords(universe, dest.item(), dest.mode(), direction, currLanguage);
        return decision.items().concat(recursive);
      }).getOr([]);

      return Arr.filter(result, function (res) {
        // Removing the unicode characters that mess up with words. This won't be sufficient, but 
        // we'll have to look at handling this later.
        return res.text().replace(Unicode.zeroWidth(), '') !== '';
      });
    };

    // Represent all the text nodes within all the sub-tree elements of item.
    // Returns: [WordDecision.make Struct] of all the words at item.
    // TODO: for TBIO-470: For multi-language spell checking: This currently assumes the language of 'item' 
    //       is the language of the entire descendent tree of 'item'. This will be wrong if the sub-tree
    //       has multiple languages annotated. Extract.all needs to return an array of items that retain this language type information.
    var extract = function (universe, item, optimise) {
      if (universe.property().isText(item)) return [ WordDecision.detail(universe, item) ];
      var children = Extract.all(universe, item, optimise);
      return Arr.bind(children, function (child) {
        return universe.property().isText(child) ? [ WordDecision.detail(universe, child) ] : [];
      });
    };

    // Returns: Option(string) of the LANG attribute of the closest ancestor element or None.
    //  - uses Fun.constant(false) for isRoot parameter to search even the top HTML element
    //    (regardless of 'classic'/iframe or 'inline'/div mode).
    // Note: there may be descendant elements with a different language
    var language = function (universe, item) {
      return universe.up().closest(item, '[lang]', Fun.constant(false)).map(function (el) {
        return universe.attrs().get(el, 'lang');
      });
    };

    // Return the words to the left and right of item, and the descendants of item (middle), and the language of item.
    var words = function (universe, item, optimise) {
      var lang = language(universe, item); // closest language anywhere up the DOM ancestor path
      var toLeft = doWords(universe, item, Gather.sidestep, WordWalking.left, lang); // lang tag of the current element, if any
      var middle = extract(universe, item, optimise); // TODO: for TBIO-470 multi-language spelling: for now we treat middle/innerText as being single language
      var toRight = doWords(universe, item, Gather.sidestep, WordWalking.right, lang); // lang tag of the current element, if any
      return {
        all: Fun.constant(Arr.reverse(toLeft).concat(middle).concat(toRight)),
        left: Fun.constant(toLeft),
        middle: Fun.constant(middle),
        right: Fun.constant(toRight),
        lang: Fun.constant(lang)
      };
    };

    return {
      words: words,
      lanugage: language
    };
  }
);<commit_msg>TBIO-2299: typo<commit_after>define(
  'ephox.robin.words.Clustering',

  [
    'ephox.bud.Unicode',
    'ephox.compass.Arr',
    'ephox.peanut.Fun',
    'ephox.phoenix.api.general.Extract',
    'ephox.phoenix.api.general.Gather',
    'ephox.robin.words.WordDecision',
    'ephox.robin.words.WordWalking'
  ],

  function (Unicode, Arr, Fun, Extract, Gather, WordDecision, WordWalking) {
    /*
     * Identification of words:
     *
     * For boundaries, stop the gathering process and do not include
     * For empty tags, stop the gathering process and do not include
     * For text nodes:
     *   a) text node has a character break, stop the gathering process and include partial
     *   b) text node has no character breaks, keep gathering and include entire node
     * For other elements, calculate the language of the closest ancestor:
     *   a) if the (destination) language has changed, stop the gathering process and do not incude
     *   b) if the (destination) language has not changed, keep gathering and do not include
     * These rules are encoded in WordDecision.decide
     * Returns: [WordDecision.make Struct] of all the words recursively from item in direction.
     */
    var doWords = function (universe, item, mode, direction, currLanguage) {
      var destination = Gather.walk(universe, item, mode, direction);
      var result = destination.map(function (dest) {
        var decision = WordDecision.decide(universe, dest.item(), direction.slicer, currLanguage);
        var recursive = decision.abort() ? [] : doWords(universe, dest.item(), dest.mode(), direction, currLanguage);
        return decision.items().concat(recursive);
      }).getOr([]);

      return Arr.filter(result, function (res) {
        // Removing the unicode characters that mess up with words. This won't be sufficient, but 
        // we'll have to look at handling this later.
        return res.text().replace(Unicode.zeroWidth(), '') !== '';
      });
    };

    // Represent all the text nodes within all the sub-tree elements of item.
    // Returns: [WordDecision.make Struct] of all the words at item.
    // TODO: for TBIO-470: For multi-language spell checking: This currently assumes the language of 'item' 
    //       is the language of the entire descendent tree of 'item'. This will be wrong if the sub-tree
    //       has multiple languages annotated. Extract.all needs to return an array of items that retain this language type information.
    var extract = function (universe, item, optimise) {
      if (universe.property().isText(item)) return [ WordDecision.detail(universe, item) ];
      var children = Extract.all(universe, item, optimise);
      return Arr.bind(children, function (child) {
        return universe.property().isText(child) ? [ WordDecision.detail(universe, child) ] : [];
      });
    };

    // Returns: Option(string) of the LANG attribute of the closest ancestor element or None.
    //  - uses Fun.constant(false) for isRoot parameter to search even the top HTML element
    //    (regardless of 'classic'/iframe or 'inline'/div mode).
    // Note: there may be descendant elements with a different language
    var language = function (universe, item) {
      return universe.up().closest(item, '[lang]', Fun.constant(false)).map(function (el) {
        return universe.attrs().get(el, 'lang');
      });
    };

    // Return the words to the left and right of item, and the descendants of item (middle), and the language of item.
    var words = function (universe, item, optimise) {
      var lang = language(universe, item); // closest language anywhere up the DOM ancestor path
      var toLeft = doWords(universe, item, Gather.sidestep, WordWalking.left, lang); // lang tag of the current element, if any
      var middle = extract(universe, item, optimise); // TODO: for TBIO-470 multi-language spelling: for now we treat middle/innerText as being single language
      var toRight = doWords(universe, item, Gather.sidestep, WordWalking.right, lang); // lang tag of the current element, if any
      return {
        all: Fun.constant(Arr.reverse(toLeft).concat(middle).concat(toRight)),
        left: Fun.constant(toLeft),
        middle: Fun.constant(middle),
        right: Fun.constant(toRight),
        lang: Fun.constant(lang)
      };
    };

    return {
      words: words,
      language: language
    };
  }
);
{'fraction_non_alphanumeric': 0.10703712138465042, 'fraction_numerical': 0.0018219084491004327, 'mean_word_length': 3.0418775885872065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5553298', 'n_tokens_mistral': 2471, 'n_tokens_neox': 2429, 'n_words': 887}