<commit_before>'use strict';

var htmlparser = require('htmlparser2');
var isObject = require('isobject');
var objectAssign = require('object-assign');

/**
 * @see https://github.com/fb55/htmlparser2/wiki/Parser-options
 */
var defaultOptions = {lowerCaseTags: false, lowerCaseAttributeNames: false};

var defaultDirectives = [{name: '!doctype', start: '<', end: '>'}];

/**
 * Parse html to PostHTMLTree
 * @param  {String} html
 * @param  {Object} [options=defaultOptions]
 * @return {PostHTMLTree}
 */
function postHTMLParser(html, options) {
    var bufArray = [],
        results = [];

    bufArray.last = function() {
        return this[this.length - 1];
    };

    function parserDirective(name, data) {
        var directives = options.directives || defaultDirectives;
        var last = bufArray.last();

        var tagName;
        for (var i = 0; i < directives.length; i++) {
            var directive = directives[i];
            var directiveText = directive.start + data + directive.end;
            var isDirective = false;

            tagName = name.toLowerCase();
            if ((directive.name instanceof RegExp) && directive.name.test(tagName)) {
                isDirective = true;
            } else if (tagName === directive.name) {<commit_msg>split directive checking per condition<commit_after>'use strict';

var htmlparser = require('htmlparser2');
var isObject = require('isobject');
var objectAssign = require('object-assign');

/**
 * @see https://github.com/fb55/htmlparser2/wiki/Parser-options
 */
var defaultOptions = {lowerCaseTags: false, lowerCaseAttributeNames: false};

var defaultDirectives = [{name: '!doctype', start: '<', end: '>'}];

/**
 * Parse html to PostHTMLTree
 * @param  {String} html
 * @param  {Object} [options=defaultOptions]
 * @return {PostHTMLTree}
 */
function postHTMLParser(html, options) {
    var bufArray = [],
        results = [];

    bufArray.last = function() {
        return this[this.length - 1];
    };

    function parserDirective(name, data) {
        var directives = options.directives || defaultDirectives;
        var last = bufArray.last();

        var tagName;
        for (var i = 0; i < directives.length; i++) {
            var directive = directives[i];
            var directiveText = directive.start + data + directive.end;
            var isDirective = false;

            tagName = name.toLowerCase();
            if ((directive.name instanceof RegExp) && directive.name.test(tagName)) {
                isDirective = true;
            }

            if (tagName === directive.name) {
{'fraction_non_alphanumeric': 0.1371561410306083, 'fraction_numerical': 0.004649360712901976, 'mean_word_length': 2.8251851851851852, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '269456', 'n_tokens_mistral': 754, 'n_tokens_neox': 719, 'n_words': 185}