<commit_before>function PaprColumnizer($elem, settings) {

   this.$elem = $elem;
   this.settings = settings;

}


PaprColumnizer.prototype.onAfterAdd = function() {

   this.prepareForColumnization();

   // columnize is implemented by subclasses
   this.columnize();

   this.afterColumnization();

   var paprcolumns = this;
   $(window).resize(function() {
      paprcolumns.$dest.find('.col').css('height', '');
      paprcolumns.equalizeColumnHeights();
   });

};


PaprColumnizer.prototype.prepareForColumnization = function() {
   // TODO: actual destination needs to be configurable
   this.$dest = $('<div />');
   this.$contents = this.$elem.contents().clone(true);
   this.$elem.empty();
   this.$elem.append(this.$dest);
};


PaprColumnizer.prototype.afterColumnization = function() {
   this.$dest.find('.col').last().addClass('lastcol').end().first().addClass('firstcol');

   this.equalizeColumnHeights();
};


PaprColumnizer.prototype.equalizeColumnHeights = function() {
   var $cols = this.$dest.find('.col'),
       max = Math.max.apply(null, $.makeArray($cols.map(function() { return $(this).height(); })));
   $cols.height(max);
   debug('set all columns to max height: ' + max);
};


PaprColumnizer.prototype.createColumn = function(i) {
   return $('<div />').attr('id', 'col-' + i).addClass('col');
};


PaprColumnizer.prototype.splitInto = function($contents, $dest, acceptanceTest) {
   var paprcolumns = this, lastIndex = 0, $leftover = false;
   $contents.each(function(index, el) {
      var $el = $(el);
      $el.detach().appendTo($dest);
      lastIndex = index;

      if (!acceptanceTest()) {
         // take $el back out of the destination and split it
         $el.detach();
         var $newDest = $el.clone(true),
             $newContents = $newDest.contents();

         $newDest.empty().appendTo($dest);

         if ($newContents.length === 1 && $newContents.get(0).nodeType === Node.TEXT_NODE && $.trim($newContents.text()) !== '') {
            // this is a text node to split
            var leftoverText = paprcolumns.splitTextInto($newContents, $newDest, acceptanceTest);
            if (leftoverText === false) {
               // TODO: we need to handle this error situation or else we'll probably have an empty node in the column
               debug('ERROR: we were not able to successfully split text');
               leftoverText = $el.text();
            }
            $leftover = $el.text(leftoverText);
            return false;
         }

         // TODO: if $newDest has an ID we need to change the ID, add the ID as a class, etc
         $leftover = $el.empty().append(paprcolumns.splitInto($newContents, $newDest, acceptanceTest));
         return false;
      }
   });

   return $('<div />')
      .append($leftover)
      .append($contents.splice(lastIndex + 1))
      .contents();
};


PaprColumnizer.prototype.splitTextInto = function($textEl, $dest, acceptanceTest) {
   var text = $textEl.text();
   var splitLoc = this.findBestSplitLocation(text, $dest, acceptanceTest);

   if (splitLoc === false) {
      return false;
   }

   $dest.text(text.substr(0, splitLoc));
   return text.substr(splitLoc);
};


PaprColumnizer.prototype.findBestSplitLocation = function(text, $dest, acceptanceTest) {<commit_msg>remove ID of elements that are split, add classes<commit_after>function PaprColumnizer($elem, settings) {

   this.$elem = $elem;
   this.settings = settings;

}


PaprColumnizer.prototype.onAfterAdd = function() {

   this.prepareForColumnization();

   // columnize is implemented by subclasses
   this.columnize();

   this.afterColumnization();

   var paprcolumns = this;
   $(window).resize(function() {
      paprcolumns.$dest.find('.col').css('height', '');
      paprcolumns.equalizeColumnHeights();
   });

};


PaprColumnizer.prototype.prepareForColumnization = function() {
   // TODO: actual destination needs to be configurable
   this.$dest = $('<div />');
   this.$contents = this.$elem.contents().clone(true);
   this.$elem.empty();
   this.$elem.append(this.$dest);
};


PaprColumnizer.prototype.prepareSplitElement = function($orig, $newEl) {
   $orig.addClass('split');
   $newEl.addClass('split');
   if ($orig.attr('id') !== undefined && $.trim($orig.attr('id')) !== '') {
      var cls = $orig.attr('id') + '-splitID';
      $orig.removeAttr('id').addClass(cls);
      $newEl.removeAttr('id').addClass(cls);
   }
};


PaprColumnizer.prototype.afterColumnization = function() {
   this.$dest.find('.col').last().addClass('lastcol').end().first().addClass('firstcol');

   this.equalizeColumnHeights();
};


PaprColumnizer.prototype.equalizeColumnHeights = function() {
   var $cols = this.$dest.find('.col'),
       max = Math.max.apply(null, $.makeArray($cols.map(function() { return $(this).height(); })));
   $cols.height(max);
   debug('set all columns to max height: ' + max);
};


PaprColumnizer.prototype.createColumn = function(i) {
   return $('<div />').attr('id', 'col-' + i).addClass('col');
};


PaprColumnizer.prototype.splitInto = function($contents, $dest, acceptanceTest) {
   var paprcolumns = this, lastIndex = 0, $leftover = false;
   $contents.each(function(index, el) {
      var $el = $(el);
      $el.detach().appendTo($dest);
      lastIndex = index;

      if (!acceptanceTest()) {
         // take $el back out of the destination and split it
         $el.detach();
         var $newDest = $el.clone(true),
             $newContents = $newDest.contents();

         $newDest.empty().appendTo($dest);

         paprcolumns.prepareSplitElement($el, $newDest);

         if ($newContents.length === 1 && $newContents.get(0).nodeType === Node.TEXT_NODE && $.trim($newContents.text()) !== '') {
            // this is a text node to split
            var leftoverText = paprcolumns.splitTextInto($newContents, $newDest, acceptanceTest);
            if (leftoverText === false) {
               // TODO: we need to handle this error situation or else we'll probably have an empty node in the column
               debug('ERROR: we were not able to successfully split text');
               leftoverText = $el.text();
            }
            $leftover = $el.text(leftoverText);
            return false;
         }

         $leftover = $el.empty().append(paprcolumns.splitInto($newContents, $newDest, acceptanceTest));
         return false;
      }
   });

   return $('<div />')
      .append($leftover)
      .append($contents.splice(lastIndex + 1))
      .contents();
};


PaprColumnizer.prototype.splitTextInto = function($textEl, $dest, acceptanceTest) {
   var text = $textEl.text();
   var splitLoc = this.findBestSplitLocation(text, $dest, acceptanceTest);

   if (splitLoc === false) {
      return false;
   }

   $dest.text(text.substr(0, splitLoc));
   return text.substr(splitLoc);
};


PaprColumnizer.prototype.findBestSplitLocation = function(text, $dest, acceptanceTest) {
{'fraction_non_alphanumeric': 0.16211135213304412, 'fraction_numerical': 0.0014461315979754157, 'mean_word_length': 3.1512605042016806, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '474341', 'n_tokens_mistral': 2135, 'n_tokens_neox': 2054, 'n_words': 438}