<commit_before><?php

class GtkdocHtml {

    // The original HTML
    private $_raw_html;

    // The parsed HTML, suitable for inclusion in a bigger page
    private $_html;

    // The description for this section, if found in the original HTML
    private $_description;

    // The mangle table for _urlMangler()
    private $_mangle_table;


    private function _urlMangler($url) {
        // A single preg_replace() cannot be used because the
        // replacement must stop at the first match
        $count = 0;
        foreach ($this->_mangle_table as $pattern => $replacement) {
            $url = preg_replace($pattern, $replacement, $url, 1, $count);
            if ($count > 0)
                break;
        }

        return $url;
    }

    private function _processDocument(DOMDocument $doc) {
        $xpath = new DOMXPath($doc);<commit_msg>Improved doc on URL mangling<commit_after><?php

class GtkdocHtml {

    // The original HTML
    private $_raw_html;

    // The parsed HTML, suitable for inclusion in a bigger page
    private $_html;

    // The description for this section, if found in the original HTML
    private $_description;

    // The mangle table for _urlMangler()
    private $_mangle_table;


    private function _urlMangler($url) {
        // A single preg_replace() call with arrays cannot be used
        // because the replacement must stop at the first match
        $count = 0;
        foreach ($this->_mangle_table as $pattern => $replacement) {
            $url = preg_replace($pattern, $replacement, $url, 1, $count);
            if ($count > 0)
                break;
        }

        return $url;
    }

    private function _processDocument(DOMDocument $doc) {
        $xpath = new DOMXPath($doc);
{'fraction_non_alphanumeric': 0.09225512528473805, 'fraction_numerical': 0.003416856492027335, 'mean_word_length': 2.3026315789473686, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3905176', 'n_tokens_mistral': 500, 'n_tokens_neox': 475, 'n_words': 176}