<commit_before>'use strict'; // XXX
import * as fs from 'fs';
import * as parse5 from 'parse5';

//
// Parse
//
export interface Document {
    node: parse5.ASTNode
    text: string
}

export function parse(htmlPath): Promise<Document> {
    return new Promise((resolve, reject) => {
        fs.readFile(htmlPath, 'utf-8', (err, text) => {
            if (err) {
                reject(err);
                return;
            }

            const node = parse5.parse(text, {
                locationInfo: true,
            });
            const doc: Document = {
                node: node,
                text: text
            };
            resolve(doc);
        });
    });
}


//
// Utils
//
export function getBody(doc: Document): parse5.ASTNode {
    for (const html of doc.node.childNodes) {
        if (html.nodeName !== 'html') {
            continue;
        }

        for (const body of html.childNodes) {
            if (body.nodeName === 'body') {
                return body;
            }
        }
    }
    
    return null;
}

export function getAttribute(node: parse5.ASTNode, attrName: string): string {
    for (const attr of node.attrs) {
        if (attr.name === attrName) {
            return attr.value;
        }
    }

    return null;
}

export function hasClassName(node: parse5.ASTNode, nodeName: string, className: string): boolean {
    if (node.nodeName !== nodeName) {
        return false;
    }
    
    let value = getAttribute(node, 'class');
    if (!value) {
        return false;
    }
    
    value = ' ' + value.replace(/\s+/g, ' ').trim() + ' ';
    return value.indexOf(' ' + className + ' ') !== -1;
}


export function getText(node: parse5.ASTNode): string {
    if (node.nodeName === '#text') {
        return node.value;
    }

    if (node.childNodes) {
        let text = '';
        for (const child of node.childNodes) {
            text += getText(child);
        }
        return text;
    }

    return '';
}

export function getHTMLText(doc: Document, nodes: parse5.ASTNode[]): string {
    const start = nodes[0].__location.startOffset;
    const end = nodes[nodes.length - 1].__location.endOffset;
    return doc.text.substring(start, end);
}<commit_msg>(splitter) whitespace cleanup<commit_after>'use strict'; // XXX
import * as fs from 'fs';
import * as parse5 from 'parse5';

//
// Parse
//
export interface Document {
    node: parse5.ASTNode
    text: string
}

export function parse(htmlPath): Promise<Document> {
    return new Promise((resolve, reject) => {
        fs.readFile(htmlPath, 'utf-8', (err, text) => {
            if (err) {
                reject(err);
                return;
            }

            const node = parse5.parse(text, {
                locationInfo: true,
            });
            const doc: Document = {
                node: node,
                text: text
            };

            resolve(doc);
        });
    });
}


//
// Utils
//
export function getBody(doc: Document): parse5.ASTNode {
    for (const html of doc.node.childNodes) {
        if (html.nodeName !== 'html') {
            continue;
        }

        for (const body of html.childNodes) {
            if (body.nodeName === 'body') {
                return body;
            }
        }
    }

    return null;
}

export function getAttribute(node: parse5.ASTNode, attrName: string): string {
    for (const attr of node.attrs) {
        if (attr.name === attrName) {
            return attr.value;
        }
    }

    return null;
}

export function hasClassName(node: parse5.ASTNode, nodeName: string, className: string): boolean {
    if (node.nodeName !== nodeName) {
        return false;
    }

    let value = getAttribute(node, 'class');
    if (!value) {
        return false;
    }

    value = ' ' + value.replace(/\s+/g, ' ').trim() + ' ';
    return value.indexOf(' ' + className + ' ') !== -1;
}


export function getText(node: parse5.ASTNode): string {
    if (node.nodeName === '#text') {
        return node.value;
    }

    if (node.childNodes) {
        let text = '';
        for (const child of node.childNodes) {
            text += getText(child);
        }
        return text;
    }

    return '';
}

export function getHTMLText(doc: Document, nodes: parse5.ASTNode[]): string {
    const start = nodes[0].__location.startOffset;
    const end = nodes[nodes.length - 1].__location.endOffset;
    return doc.text.substring(start, end);
}
{'fraction_non_alphanumeric': 0.13678290213723285, 'fraction_numerical': 0.005849268841394826, 'mean_word_length': 1.8246505717916137, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6573883', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1306, 'n_words': 334}