<commit_before>define(
  'ephox.robin.words.Cluster',

  [
    'ephox.compass.Arr',
    'ephox.peanut.Fun',
    'ephox.perhaps.Option',
    'ephox.polaris.api.Arrays',
    'ephox.robin.api.general.Zone',
    'ephox.robin.words.Clustering',
    'ephox.robin.words.Identify',
    'ephox.sugar.api.Node',
    'ephox.sugar.api.PredicateFilter',
    'ephox.sugar.api.Text'
  ],

  function (Arr, Fun, Option, Arrays, Zone, Clustering, Identify, Node, PredicateFilter, Text) {
    /**
     * Finds words in groups of text (each HTML text node can have multiple words).
     */
    var findWords = function (universe, cluster) {
      var groups = Arrays.splitby(cluster, function (c) {
        // I really don't think this can happen ... given that the cluster is specifically excluding these.
        var elem = c.item();
        return universe.property().isBoundary(elem) || universe.property().isEmptyTag(elem);
      });

      return Arr.bind(groups, function (x) {
        var text = Arr.map(x, function (y) {
          return y.text();
        }).join('');
        return Identify.words(text);
      });
    };

    var block = function (universe, element) {

      // Will have to group this later.
      var nodes = PredicateFilter.descendants(element, Node.isText);
      var text = Arr.map(nodes, Text.get).join('');

      var words = Identify.words(text);

      return {<commit_msg>TBIO-3819: basic walking to get all text nodes without grouping them<commit_after>define(
  'ephox.robin.words.Cluster',

  [
    'ephox.compass.Arr',
    'ephox.peanut.Fun',
    'ephox.perhaps.Option',
    'ephox.phoenix.api.dom.DomGather',
    'ephox.phoenix.api.general.Gather',
    'ephox.polaris.api.Arrays',
    'ephox.robin.api.general.Zone',
    'ephox.robin.words.Clustering',
    'ephox.robin.words.Identify',
    'ephox.sugar.api.Compare',
    'ephox.sugar.api.Node',
    'ephox.sugar.api.PredicateFilter',
    'ephox.sugar.api.Text'
  ],

  function (Arr, Fun, Option, DomGather, Gather, Arrays, Zone, Clustering, Identify, Compare, Node, PredicateFilter, Text) {
    /**
     * Finds words in groups of text (each HTML text node can have multiple words).
     */
    var findWords = function (universe, cluster) {
      var groups = Arrays.splitby(cluster, function (c) {
        // I really don't think this can happen ... given that the cluster is specifically excluding these.
        var elem = c.item();
        return universe.property().isBoundary(elem) || universe.property().isEmptyTag(elem);
      });

      return Arr.bind(groups, function (x) {
        var text = Arr.map(x, function (y) {
          return y.text();
        }).join('');
        return Identify.words(text);
      });
    };

    var block = function (universe, element) {

      // Will have to group this later.
      var _rules = undefined;
      var nodes = [ ];

      var walk = function (item, mode) {
        return DomGather.walk(item, mode, DomGather.walkers().right(), _rules).map(function (n) {
          if (Compare.eq(n.item(), element)) return [ ];
          var self = Node.isText(n.item()) ? [ n.item() ] : [ ];
          var rest = walk(n.item(), n.mode());
          return self.concat(rest);
        }).getOr([ ]);
      };

      var allnodes = walk(element, Gather.advance);
      console.log('allnodes', Arr.map(allnodes, Text.get));

      var nodes = PredicateFilter.descendants(element, Node.isText);
      var text = Arr.map(nodes, Text.get).join('');

      var words = Identify.words(text);

      return {
{'fraction_non_alphanumeric': 0.14919011082693948, 'fraction_numerical': 0.0011366865586814436, 'mean_word_length': 2.595505617977528, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6069615', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1149, 'n_words': 264}