<commit_before><?php
/**
 * HTMLPage
 * @package HTML
 */

if (!class_exists('DOMDocument')) {
    die('DOMDocument Functions not available (php-xml)');
}

if (!function_exists('mb_convert_encoding')) {
    die('Multibyte String Functions not available (mbstring)');
}

/**
 * HTMLPage
 * @package HTML
 */
class HTMLPage {
  private $document;
  private $body;
  private static $header = "<html><body>";
  private static $footer = "</body></html>";
  
  public function __construct() {
    $this->document = new DOMDocument();
    $this->document->loadHTML(self::$header . self::$footer);
    $root = $this->document;
    $this->body = $root->getElementsByTagName("body")->item(0);
  }
  
  public function addNode(DOMNode $node) {
    $nodeCopy = $this->document->importNode($node, true);
    $this->body->appendChild($nodeCopy);
  }
  
  public function getText() {
    $text = $this->document->saveHTML();
    
    // removes header
    $content_position = strpos($text, self::$header) + strlen(self::$header);
    $text = substr($text, $content_position);
    
    // removes footer
    $text = substr($text, 0, strlen($text) - strlen(self::$footer)-1);
    
    return trim($text);
  }

}

/**
 * HTMLPager
 * @package HTML
 */
class HTMLPager {
  const PARAGRAPH_LIMIT=4;
  const ALL_PAGES='all';
  private $pages = array();
  private $pageCount = 0;
  private $pageNumber = 0;
  
  public function __construct($html, $encoding, $pageNumber, $paragraphsPerPage=HTMLPager::PARAGRAPH_LIMIT) {
    $dom = new DOMDocument();
    $dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', $encoding));
    $body = $dom->getElementsByTagName("body")->item(0);

    $currentPage = NULL;
    $pages = array();
    $currentParagraphCount = 0;

    foreach($body->childNodes as $node) {
      if($currentPage == NULL) {
        // need to start a new page
        if(($node->nodeName == "#text") && (trim($node->nodeValue) == "")) {
          continue; // this node is blank so do not start a new page yet
        }

        $currentPage = new HTMLPage();
        $pages[] = $currentPage;
      }

      $currentPage->addNode($node);
<commit_msg>Move warnings generated	by loadHTML into error log, not	spewed into the	web page since we don't have control over the feed content (not much a developer	can do to fix it)<commit_after><?php
/**
 * HTMLPage
 * @package HTML
 */

if (!class_exists('DOMDocument')) {
    die('DOMDocument Functions not available (php-xml)');
}

if (!function_exists('mb_convert_encoding')) {
    die('Multibyte String Functions not available (mbstring)');
}

/**
 * HTMLPage
 * @package HTML
 */
class HTMLPage {
  private $document;
  private $body;
  private static $header = "<html><body>";
  private static $footer = "</body></html>";
  
  public function __construct() {
    $this->document = new DOMDocument();
    $this->document->loadHTML(self::$header . self::$footer);
    $root = $this->document;
    $this->body = $root->getElementsByTagName("body")->item(0);
  }
  
  public function addNode(DOMNode $node) {
    $nodeCopy = $this->document->importNode($node, true);
    $this->body->appendChild($nodeCopy);
  }
  
  public function getText() {
    $text = $this->document->saveHTML();
    
    // removes header
    $content_position = strpos($text, self::$header) + strlen(self::$header);
    $text = substr($text, $content_position);
    
    // removes footer
    $text = substr($text, 0, strlen($text) - strlen(self::$footer)-1);
    
    return trim($text);
  }

}

/**
 * HTMLPager
 * @package HTML
 */
class HTMLPager {
  const PARAGRAPH_LIMIT=4;
  const ALL_PAGES='all';
  private $pages = array();
  private $pageCount = 0;
  private $pageNumber = 0;
  
  public function __construct($html, $encoding, $pageNumber, $paragraphsPerPage=HTMLPager::PARAGRAPH_LIMIT) {
    $dom = new DOMDocument();
    
    libxml_use_internal_errors(true);
    libxml_clear_errors(); // clean up any errors belonging to other operations
    $dom->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', $encoding));
    foreach (libxml_get_errors() as $error) {
      error_log("HTMLPager got loadHTML warning (line {$error->line}; column {$error->column}) {$error->message}");
    }
    libxml_clear_errors(); // free up memory associated with the errors
    libxml_use_internal_errors(false);
    
    $body = $dom->getElementsByTagName("body")->item(0);

    $currentPage = NULL;
    $pages = array();
    $currentParagraphCount = 0;

    foreach($body->childNodes as $node) {
      if($currentPage == NULL) {
        // need to start a new page
        if(($node->nodeName == "#text") && (trim($node->nodeValue) == "")) {
          continue; // this node is blank so do not start a new page yet
        }

        $currentPage = new HTMLPage();
        $pages[] = $currentPage;
      }

      $currentPage->addNode($node);

{'fraction_non_alphanumeric': 0.16123711340206184, 'fraction_numerical': 0.0032989690721649486, 'mean_word_length': 3.4545454545454546, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7011943', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1522, 'n_words': 363}