<commit_before>/*jslint indent: 2, node: true*/
/*globals document*/
"use strict";

var phantomWrapper = require("./phantom-wrapper.js"),
  template = require("./template.js");

exports.createPage = function (blankPage, content, callback) {
  var page,
    pageMarkup = "",
    leftoverMarkup = "";

  function gotContent(err, pageMarkup) {
    if (err) {
      callback(err);
      return;
    }

    callback(null, pageMarkup, leftoverMarkup);
  }

  function filledPage(err, leftover) {
    if (err) {
      callback(err);
      return;
    }

    if (leftover && leftover.indexOf("Error: ") === 0) {
      callback(leftover);
      return;
    }

    leftoverMarkup = leftover;

    page.get("content", gotContent);
  }

  function setContent(err) {
    if (err) {
      callback(err);
      return;
    }

    page.evaluate(function (markup) {
      var leftover = "",
        lastElement,
        container,
        body;

      function removeParagraph() {
        var lastWordIndex;

        // return if container doesn't overflow anymore
        if (container.clientHeight >= container.scrollHeight) {
          return;
        }

        lastElement = container.lastChild;
        container.removeChild(lastElement);

        // if it's a non-element node, ignore it
        if (lastElement.nodeType !== 1) {
          removeParagraph();
          return;
        }

        // if this last paragraph was the difference between container overflowing or not
        if (container.clientHeight >= container.scrollHeight
            && lastElement.tagName === "P"
            && lastElement.innerHTML.indexOf(" ") > -1) { // ignore single-word paragraphs
          // readd it...
          container.appendChild(lastElement);

          // prep leftover to wrap paragraph
          leftover = "</p>" + leftover;

          // and remove words one by one until it stops overflowing again
          while (container.clientHeight < container.scrollHeight) {
            lastWordIndex = lastElement.innerHTML.lastIndexOf(" ");
            leftover = lastElement.innerHTML.substring(lastWordIndex + 1) + " " + leftover;
            lastElement.innerHTML = lastElement.innerHTML.substring(0, lastWordIndex);
          }

          if (lastElement.innerHTML.length > 0) {
            // contd class should have an indent 0
            leftover = "<p class='contd " + lastElement.className + "'>" + leftover;
          } else {
            leftover = "<p>" + leftover;
          }
        } else {
          leftover = lastElement.outerHTML + leftover;
          removeParagraph();
        }
      }

      try {
        // add all of the content to the container element
        body = document.body;
        container = document.getElementById("container");
        container.innerHTML = markup;

        // remove paragraphs until it doesn't overflow anymore
        removeParagraph();
      } catch (e) {
        return "Error: " + e.toString();
      }

      return leftover; // to be populated with leftover text
    }, filledPage, content);
  }
<commit_msg>attempting to make paginator recursive in order to parse inline elements<commit_after>/*jslint indent: 2, node: true*/
/*globals document*/
"use strict";

var phantomWrapper = require("./phantom-wrapper.js"),
  template = require("./template.js");

exports.createPage = function (blankPage, content, callback) {
  var page,
    pageMarkup = "",
    leftoverMarkup = "";

  function gotContent(err, pageMarkup) {
    if (err) {
      callback(err);
      return;
    }

    callback(null, pageMarkup, leftoverMarkup);
  }

  function filledPage(err, leftover) {
    if (err) {
      callback(err);
      return;
    }

    if (leftover && leftover.indexOf("Error: ") === 0) {
      callback(leftover);
      return;
    }

    leftoverMarkup = leftover;

    page.get("content", gotContent);
  }

  function setContent(err) {
    if (err) {
      callback(err);
      return;
    }

    page.evaluate(function (markup) {
      var leftover = "",
        container,
        body,
        splittableTags = ["P", "A", "SPAN", "EM", "STRONG"];

      function removeOverflowingElements(parentElement) {
        var lastWordIndex,
          lastNode,
          lastWord,
          parentTagName;

        // nothing to do if top level container's stopped overflowing
        if (container.clientHeight >= container.scrollHeight) {
          return;
        }

        lastNode = parentElement.lastChild;
        parentElement.removeChild(lastNode);

        parentTagName = parentElement.tagName.toLowerCase();

        // if this last element was the difference between container overflowing
        // it may need to be split
        if (container.clientHeight >= container.scrollHeight) {

          if (lastNode.nodeType === 1 // if node is type element (ie a tag)
              && splittableTags.indexOf(lastNode.tagName) > -1 // only split P and inline tags
              && lastNode.innerHTML.indexOf(" ") > -1) { // can't split one word elements so don't bother

            // readd it...
            parentElement.appendChild(lastNode);

            // and recurse
            removeOverflowingElements(lastNode);

          } else if (lastNode.nodeType === 3 // if node is text element
              && lastNode.data) { // and is non-empty

            // readd it...
            parentElement.appendChild(lastNode);

            // prep leftover to wrap
            leftover = "</" + parentTagName + ">" + leftover;

            // and remove words one by one until it stops overflowing again
            while (container.clientHeight < container.scrollHeight) {
              lastWordIndex = lastNode.data.lastIndexOf(" ");
              lastWord = lastNode.data.substring(lastWordIndex + 1);

              leftover = lastWord + " " + leftover;
              lastNode.data = lastNode.data.substring(0, lastWordIndex);
            }

            if (lastNode.data.length > 0) {
              // contd class should have an indent 0
              leftover = "<" + parentTagName + " class='contd " + parentElement.className + "'>" + leftover;
            } else {
              leftover = "<" + parentTagName + ">" + leftover;
            }
          } else {
            leftover = lastNode.outerHTML + leftover;
          }

        } else if (lastNode.outerHTML) {
          leftover = lastNode.outerHTML + leftover;
        }

        removeOverflowingElements(parentElement);
      }

      try {
        // add all of the content to the container element
        body = document.body;
        container = document.getElementById("container");
        container.innerHTML = markup;

        // remove paragraphs until it doesn't overflow anymore
        removeOverflowingElements(container);
      } catch (e) {
        return "Error: " + e.toString();
      }

      return leftover; // to be populated with leftover text
    }, filledPage, content);
  }

{'fraction_non_alphanumeric': 0.11154401154401154, 'fraction_numerical': 0.0025974025974025974, 'mean_word_length': 2.0345884413309983, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1489469', 'n_tokens_mistral': 1994, 'n_tokens_neox': 1822, 'n_words': 485}