<commit_before>/**
    @file HTML element implementation.
*/
"use strict";

/**
    @class HTMLElement
    @classdesc Fairly simple HTML element implementation, that includes
        necessary properties.

    @property dataType {String} The type of item that this instance contains.
    @property content {String} The string contents of the item.
*/
function HTMLElement(dataType, content) {
    this.dataType = dataType;
    if (content) {
        this.content = content;
    }
}

/**
    Represent different types of behaviors that need to be used for the types
    of elements we'll encounter.

    @property Text - Arbitrary text.
    @property Tag - Any ordinary HTML tag.
    @property Comment - Commented out HTML. Essentially a Text element.
*/
HTMLElement.DataTypes = {
    Text: "text",
    Tag: "tag",
    Comment: "comment",
    Attribute: "attribute"
};

/**
    Regular expressions that are needed throughout.

    @property Text {RegExp} Strip out all text outside of HTML tags.
    @property Tag {RegExp} Extract all contents of a tag, including attributes.
    @property AttributeName {RegExp} Get the name of an attribute.
    @property AttributeValye {RegExp} Get the value of an attribute after
    getting rid of its name.
    @property Comment {RegExp} Strip out the contents of a comment.
    @property XPath {Object} A few XPath related RegExps.
    @property FullTag {RegExp} A really long regular expression that extracts
    the first complete tag and attributes from the input.
*/
HTMLElement.RegExp = {
    Text: /[^]+?((?=\<([a-zA-Z_]|\!--))|$)/,
    AttributeName: /[^\s\=\>]+/,
    AttributeValue: /^([^\"\'\>\s]+|\"[^\"]*\"|\'[^\']*\')/,
    Comment: /^\<!--[^]+?(?=--\>)--\>/,
    XPath: {
        ChildNodeSearch: /^\/(@?[\w\-]+|\*)/,
        AnyNodeSearch: /^\/\/(@?[\w\-]+|\*)/,
        Predicate: /^\[([^\]]+)\]/
    }
};

HTMLElement.RegExp.FullTag = new RegExp(
    "^\\<\\s*" + // Tag open
    HTMLElement.RegExp.AttributeName.source + // Tag Name
    "(" + // Optional Attribute list.
        "(\\s*" +
            HTMLElement.RegExp.AttributeName.source + // Attribute Name.
        "\\s*)" +
        "(\\s*\\=\\s*" + // If there's a "value" there's an =.
            "(" +
                HTMLElement.RegExp.AttributeValue.source.substring(1) +
            ")?" + // Optional string value within the whole value.
        "\\s*)?" + // Or one with a value.
    ")*" +
    "\\s*\\>" // Tag close.
);
<commit_msg>Removed superfluous whitespace in tag extractor.<commit_after>/**
    @file HTML element implementation.
*/
"use strict";

/**
    @class HTMLElement
    @classdesc Fairly simple HTML element implementation, that includes
        necessary properties.

    @property dataType {String} The type of item that this instance contains.
    @property content {String} The string contents of the item.
*/
function HTMLElement(dataType, content) {
    this.dataType = dataType;
    if (content) {
        this.content = content;
    }
}

/**
    Represent different types of behaviors that need to be used for the types
    of elements we'll encounter.

    @property Text - Arbitrary text.
    @property Tag - Any ordinary HTML tag.
    @property Comment - Commented out HTML. Essentially a Text element.
*/
HTMLElement.DataTypes = {
    Text: "text",
    Tag: "tag",
    Comment: "comment",
    Attribute: "attribute"
};

/**
    Regular expressions that are needed throughout.

    @property Text {RegExp} Strip out all text outside of HTML tags.
    @property Tag {RegExp} Extract all contents of a tag, including attributes.
    @property AttributeName {RegExp} Get the name of an attribute.
    @property AttributeValye {RegExp} Get the value of an attribute after
    getting rid of its name.
    @property Comment {RegExp} Strip out the contents of a comment.
    @property XPath {Object} A few XPath related RegExps.
    @property FullTag {RegExp} A really long regular expression that extracts
    the first complete tag and attributes from the input.
*/
HTMLElement.RegExp = {
    Text: /[^]+?((?=\<([a-zA-Z_]|\!--))|$)/,
    AttributeName: /[^\s\=\>]+/,
    AttributeValue: /^([^\"\'\>\s]+|\"[^\"]*\"|\'[^\']*\')/,
    Comment: /^\<!--[^]+?(?=--\>)--\>/,
    XPath: {
        ChildNodeSearch: /^\/(@?[\w\-]+|\*)/,
        AnyNodeSearch: /^\/\/(@?[\w\-]+|\*)/,
        Predicate: /^\[([^\]]+)\]/
    }
};

HTMLElement.RegExp.FullTag = new RegExp(
    "^\\<\\s*" + // Tag open
    HTMLElement.RegExp.AttributeName.source + // Tag Name
    "(" + // Optional Attribute list.
        "(\\s*" +
            HTMLElement.RegExp.AttributeName.source + // Attribute Name.
        "\\s*)" +
        "(\\=\\s*" + // If there's a "value" there's an =.
            "(" +
                HTMLElement.RegExp.AttributeValue.source.substring(1) +
            ")?" + // Optional string value within the whole value.
        "\\s*)?" + // Or one with a value.
    ")*" +
    "\\s*\\>" // Tag close.
);

{'fraction_non_alphanumeric': 0.16636308439587127, 'fraction_numerical': 0.0004047763610605141, 'mean_word_length': 2.429562803608605, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1261509', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1505, 'n_words': 479}