<commit_before>/*jslint indent: 2, node: true*/
/*globals it, describe, beforeEach, before*/
"use strict";

var should = require("should"),
  html = require("../../html.js"),
  phantomWrapper = require("../../phantom-wrapper.js"),
  lipsum = require("lorem-ipsum"),
  helper = require("../helper.js"),
  dimensions;

describe("html", function () {
  describe("#createPage()", function () {
    var lipsumOptions,
      markup,
      emptyPageMarkup;

    beforeEach(function () {
      dimensions = require("../../dimensions.js");
      lipsumOptions = {
        format: "html",
        units: "paragraph"
      };
      emptyPageMarkup = html.getNewEmptyPageMarkup();
    });
    after(function (done) {
      phantomWrapper.cleanup(function () {
        helper.killProcess("phantomjs", done)
      });
    });

    it("should output a filled out page", function (done) {
      lipsumOptions.count = 1;
      markup = lipsum(lipsumOptions);
      html.createPage(emptyPageMarkup, markup, function (err, page, leftoverMarkup) {
        should.not.exist(err);
        should.exist(page);
        should.exist(leftoverMarkup);
        page.should.not.equal(emptyPageMarkup);
        leftoverMarkup.should.be.empty;
        done();
      });
    });
    it("should output a page and leftovers if passed a long text", function (done) {
      lipsumOptions.count = 100;
      markup = lipsum(lipsumOptions);
      html.createPage(emptyPageMarkup, markup, function (err, page, leftoverMarkup) {
        should.not.exist(err);
        should.exist(leftoverMarkup);
        leftoverMarkup.should.not.be.empty;
        done();
      });
    });
    it("should be able to split paragraphs at page break", function (done) {
      var i, wordsInPage, wordsInLeftover, numberWords = 500;
      markup = "<p>start ";
      for (i = 0; i < numberWords; i += 1) {
        markup += "word ";
      }
      markup += "end</p>";
      html.createPage(emptyPageMarkup, markup, function (err, page, leftoverMarkup) {
        should.not.exist(err);
        should.exist(page);
        should.exist(leftoverMarkup);
        // "word" should be the first word in leftover
        leftoverMarkup.indexOf("word").should.equal(0);

        // check no words have been left out
        wordsInPage = (page.match(/word/g) || []).length;
        wordsInLeftover = (leftoverMarkup.match(/word/g) || []).length;
        (wordsInPage + wordsInLeftover).should.equal(numberWords);
        done();
      });
    });
  });
  describe("#paginate()", function () {
    var lipsumOptions,
      markup;
    beforeEach(function () {
      dimensions = require("../../dimensions.js");
      lipsumOptions = {
        format: "html",
        units: "paragraph"
      };
    });
    it("should output a single page if the content is small enough to fit", function (done) {
      lipsumOptions.count = 1;
      lipsumOptions.units = "sentence";
      markup = lipsum(lipsumOptions);

      html.paginate({ bodyMarkup: markup }, function (err, pages) {
        should.not.exist(err);
        pages.should.have.length(1);
        done();
      });
    });
    it("should output multiple pages if the content is larger", function (done) {
      lipsumOptions.count = 10;
      markup = lipsum(lipsumOptions);

      html.paginate({ bodyMarkup: markup }, function (err, pages) {
        should.not.exist(err);
        pages.length.should.be.above(1);
        done();
      });
    });
    it("should output pages with full head and body tags", function (done) {
      lipsumOptions.count = 10;
      markup = lipsum(lipsumOptions);

      html.paginate({ bodyMarkup: markup }, function (err, pages) {
        should.not.exist(err);
        pages.forEach(function (page) {
          page.should.include("<html>");
          page.should.include("<head>");
          page.should.include("</head>");
          page.should.include("<body>");
          page.should.include("</body>");
        });
        done();
      });
    });
    it("should not leave any content overflowing from the page when images are not involved");
    it("should not split self-closing tags");
    it("should close and reopen paragraph tags if it splits a paragraph");
    it("should keep the images on the same page as the text they are around");
    it("should not leave any content overflowing from the page when images are involved");
  });
});
<commit_msg>fixes to unit html tests<commit_after>/*jslint indent: 2, node: true*/
/*globals it, describe, beforeEach, before*/
"use strict";

var should = require("should"),
  html = require("../../html.js"),
  phantomWrapper = require("../../phantom-wrapper.js"),
  lipsum = require("lorem-ipsum"),
  helper = require("../helper.js"),
  dimensions;

describe("html", function () {
  after(function (done) {
    phantomWrapper.cleanup(function () {
      helper.killProcess("phantomjs", done)
    });
  });
  describe("#createPage()", function () {
    var lipsumOptions,
      markup,
      emptyPageMarkup;

    beforeEach(function () {
      dimensions = require("../../dimensions.js");
      lipsumOptions = {
        format: "html",
        units: "paragraph"
      };
      emptyPageMarkup = html.getNewEmptyPageMarkup();
    });

    it("should output a filled out page", function (done) {
      lipsumOptions.count = 1;
      markup = lipsum(lipsumOptions);
      html.createPage(emptyPageMarkup, markup, function (err, page, leftoverMarkup) {
        should.not.exist(err);
        should.exist(page);
        should.exist(leftoverMarkup);
        page.should.not.equal(emptyPageMarkup);
        leftoverMarkup.should.be.empty;
        done();
      });
    });
    it("should output a page and leftovers if passed a long text", function (done) {
      lipsumOptions.count = 100;
      markup = lipsum(lipsumOptions);
      html.createPage(emptyPageMarkup, markup, function (err, page, leftoverMarkup) {
        should.not.exist(err);
        should.exist(leftoverMarkup);
        leftoverMarkup.should.not.be.empty;
        done();
      });
    });
    it("should be able to split paragraphs at page break", function (done) {
      var i, wordsInPage, wordsInLeftover, numberWords = 500;
      markup = "<p>start ";
      for (i = 0; i < numberWords; i += 1) {
        markup += "word ";
      }
      markup += "end</p>";
      html.createPage(emptyPageMarkup, markup, function (err, page, leftoverMarkup) {
        should.not.exist(err);
        should.exist(page);
        should.exist(leftoverMarkup);
        // "word" should be the first word in leftover
        leftoverMarkup.indexOf("word").should.equal(0);

        // check no words have been left out
        wordsInPage = (page.match(/word/g) || []).length;
        wordsInLeftover = (leftoverMarkup.match(/word/g) || []).length;
        (wordsInPage + wordsInLeftover).should.equal(numberWords);
        done();
      });
    });
  });
  describe("#paginate()", function () {
    var lipsumOptions,
      markup;
    beforeEach(function () {
      dimensions = require("../../dimensions.js");
      lipsumOptions = {
        format: "html",
        units: "paragraph"
      };
    });
    it("should output a single page if the content is small enough to fit", function (done) {
      lipsumOptions.count = 1;
      lipsumOptions.units = "sentence";
      markup = lipsum(lipsumOptions);

      html.paginate({ bodyMarkup: markup }, function (err, pages) {
        should.not.exist(err);
        pages.should.have.length(1);
        done();
      });
    });
    it("should output multiple pages if the content is larger", function (done) {
      lipsumOptions.count = 10;
      markup = lipsum(lipsumOptions);

      html.paginate({ bodyMarkup: markup }, function (err, pages) {
        should.not.exist(err);
        pages.length.should.be.above(1);
        done();
      });
    });
    it("should output pages with full head and body tags", function (done) {
      lipsumOptions.count = 10;
      markup = lipsum(lipsumOptions);

      html.paginate({ bodyMarkup: markup }, function (err, pages) {
        should.not.exist(err);
        pages.forEach(function (page) {
          page.should.include("<html>");
          page.should.include("<head>");
          page.should.include("</head>");
          page.should.include("<body");
          page.should.include("</body>");
        });
        done();
      });
    });
  });
});

{'fraction_non_alphanumeric': 0.14897423664122136, 'fraction_numerical': 0.004293893129770993, 'mean_word_length': 2.5245901639344264, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4287957', 'n_tokens_mistral': 2596, 'n_tokens_neox': 2448, 'n_words': 579}