<commit_before>/**
 * @file Xml Parser
 * @author <NAME> <<EMAIL>>
 * @private
 */


import { Debug, Log, ParserRegistry } from "../globals.js";
import { defaults } from "../utils.js";
import { parseXml } from "../utils/parse-xml.js";
import Parser from "./parser.js";


class XmlParser extends Parser{

    constructor( streamer, params ){

        const p = params || {};

        super( streamer, p );

        this.useDomParser = defaults( p.useDomParser, false );

        this.xml = {
            name: this.name,
            path: this.path,
            data: {}
        };

    }

    get type (){ return "xml"; }
    get __objName (){ return "xml"; }

    __xmlParser( xml ){

        return parseXml( xml );

    }

    __domParser( xml ){

        const domParser = new window.DOMParser();
        return domParser.parseFromString( xml, "text/xml" );

    }

    _parse(){

        if( Debug ) Log.time( "XmlParser._parse " + this.name );

        if( this.useDomParser ){
            if( this.streamer.isBinary() || this.string ){
                this.xml.data = this.__domParser( this.streamer.asText() );
            }else{
                this.xml.data = this.streamer.data;
            }
        }else{
            this.xml.data = this.__xmlParser( this.streamer.asText() );
        }

        if( Debug ) Log.timeEnd( "XmlParser._parse " + this.name );

    }

}

ParserRegistry.add( "xml", XmlParser );


export default XmlParser;<commit_msg>more robust xml parser<commit_after>/**
 * @file Xml Parser
 * @author <NAME> <<EMAIL>>
 * @private
 */


import { Debug, Log, ParserRegistry } from "../globals.js";
import { defaults } from "../utils.js";
import { parseXml } from "../utils/parse-xml.js";
import Parser from "./parser.js";


class XmlParser extends Parser{

    constructor( streamer, params ){

        const p = params || {};

        super( streamer, p );

        this.useDomParser = defaults( p.useDomParser, false );

        this.xml = {
            name: this.name,
            path: this.path,
            data: {}
        };

    }

    get type (){ return "xml"; }
    get __objName (){ return "xml"; }

    __xmlParser( xml ){

        return parseXml( xml );

    }

    __domParser( xml ){

        const domParser = new window.DOMParser();
        return domParser.parseFromString( xml, "text/xml" );

    }

    _parse(){

        if( Debug ) Log.time( "XmlParser._parse " + this.name );

        if( this.useDomParser ){
            if( this.streamer.data instanceof Document ){
                this.xml.data = this.streamer.data;
            }else{
                this.xml.data = this.__domParser( this.streamer.asText() );
            }
        }else{
            this.xml.data = this.__xmlParser( this.streamer.asText() );
        }

        if( Debug ) Log.timeEnd( "XmlParser._parse " + this.name );

    }

}

ParserRegistry.add( "xml", XmlParser );


export default XmlParser;
{'fraction_non_alphanumeric': 0.14515027322404372, 'fraction_numerical': 0.0, 'mean_word_length': 1.9260739260739261, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 31}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2454001', 'n_tokens_mistral': 930, 'n_tokens_neox': 868, 'n_words': 211}