<commit_before>const jsdom = require('jsdom');
const serializeDocument = jsdom.serializeDocument;
const mathjax = require('mathjax-node');
const typeset = mathjax.typeset;
const tex = require('./tex.js').tex2jax;
const ascii = require('./ascii.js').ascii2jax;
const mathml = require('./mathml.js').mathml2jax;
// HACK isolate confingurations cf #22
const texconfig = JSON.stringify(tex.config);
const asciiconfig = JSON.stringify(ascii.config);


exports.mjpage = function(htmlstring, configOptions, typesetOptions, callback) {

    // `config` extends options for mathjax-node's config method, cf https://github.com/mathjax/MathJax-node/wiki/Configuration-options#configoptions

    const config = {
        // mathjax-node-page specific
        format: ["MathML", "TeX", "AsciiMath"], // determines type of pre-processors to run
        tex: {}, // configuration options for tex pre-processor
        ascii: {}, // configuration options for ascii pre-processor
        singleDollars: false, // allow single-dollar delimiter for inline TeX
        fragment: false,
        jsdom: {
            FetchExternalResources: [],
            ProcessExternalResources: false,
            // NOTE virtualConsole: not the jsdom way but for configurability (cf. below)
            virtualConsole: true
        },
        //
        // standard mathjax-node options
        //
        displayMessages: false, // determines whether Message.Set() calls are logged
        displayErrors: true, // determines whether error messages are shown on the console
        undefinedCharError: false, // determines whether unknown characters are saved in the error array
        extensions: '', // a convenience option to add MathJax extensions
        fontURL: 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js/fonts/HTML-CSS', // for webfont urls in the CSS for HTML output
        MathJax: {} // options MathJax configuration, see https://docs.mathjax.org
    }
    const mjstate = {};
    // defaults for mathjax-node's typeset method
    const typesetConfig = {
        ex: 6, // ex-size in pixels
        width: 100, // width of container (in ex) for linebreaking and tags
        useFontCache: true, // use <defs> and <use> in svg output?
        useGlobalCache: false, // use common <defs> for all equations?
        state: mjstate, // track global state
        linebreaks: false, // do linebreaking?
        equationNumbers: "none", // or "AMS" or "all"
        math: "", // the math to typeset
        html: false, // generate HTML output?
        css: false, // generate CSS for HTML output?
        mml: false, // generate mml output?
        svg: false, // generate svg output?
        speakText: true, // add spoken annotations to svg output?
        speakRuleset: "mathspeak", // set speech ruleset (default (chromevox rules), mathspeak)
        speakStyle: "default", // set speech style (mathspeak:  default, brief, sbrief)
        timeout: 10 * 1000, // 10 second timeout before restarting MathJax
    }


    //merge configurations

    const mergeConfig = function(conf, confDefaults) {
        for (index in conf) {
            confDefaults[index] = conf[index];
        }
    };

    mergeConfig(configOptions, config);
    mergeConfig(typesetOptions, typesetConfig);

    // Set jsdom console (cf. defaults for config.jsdom)
    if (config.jsdom.virtualConsole) {
        config.jsdom.virtualConsole = jsdom.createVirtualConsole().sendTo(console);
    }

    // set up DOM basics
    const doc = jsdom.jsdom(htmlstring, config.jsdom);
    const window = doc.defaultView;
    const document = window.document;

    // configure mathjax-node
    mathjax.config(config);

    // configure and pre-process
    if (config.format.indexOf('MathML') > -1) {
        window.mathml = mathml;
        window.mathml.config.doc = document;
        window.mathml.PreProcess();
    }
    // HACK clone configuration cf #22
    tex.config = JSON.parse(texconfig);
    if (config.format.indexOf('TeX') > -1) {
        if (config.MathJax.tex2jax) {
            mergeConfig(config.MathJax.tex2jax, tex.config);
        }
        window.tex = tex;
        window.tex.config.doc = document;
        if (config.singleDollars) {
            window.tex.config.inlineMath.push(['$', '$']);
            window.tex.config.processEscapes = true;
        }
        window.tex.PreProcess();
    }
    // HACK clone configuration cf #22
    ascii.config = JSON.parse(asciiconfig);
    if (config.format.indexOf('AsciiMath') > -1) {
        if (config.MathJax.ascii2jax) {
            mergeConfig(config.MathJax.ascii2jax, ascii.config);
        }
        window.ascii = ascii;
        window.ascii.config.doc = document;
        window.ascii.PreProcess();
    }
    //normalize type strings
    const scripts = document.querySelectorAll('script[type^="math/"]');
    for (let i = 0; i < scripts.length; i++) {
        const script = scripts[i];
        let format = script.getAttribute("type").toLowerCase().slice(5);
        if (format === "tex" || "inline-tex") {
            format = "math/inline-TeX";
            // TODO Should use match to handle arbitrary spaces that are supported by MathJax
            // https://github.com/mathjax/MathJax/blob/master/unpacked/jax/input/TeX/jax.js#L2168
        } else if (format === "tex; mode=display") {
            format = "math/TeX";<commit_msg>Fixed broken test<commit_after>const jsdom = require('jsdom');
const serializeDocument = jsdom.serializeDocument;
const mathjax = require('mathjax-node');
const typeset = mathjax.typeset;
const tex = require('./tex.js').tex2jax;
const ascii = require('./ascii.js').ascii2jax;
const mathml = require('./mathml.js').mathml2jax;
// HACK isolate confingurations cf #22
const texconfig = JSON.stringify(tex.config);
const asciiconfig = JSON.stringify(ascii.config);


exports.mjpage = function(htmlstring, configOptions, typesetOptions, callback) {

    // `config` extends options for mathjax-node's config method, cf https://github.com/mathjax/MathJax-node/wiki/Configuration-options#configoptions

    const config = {
        // mathjax-node-page specific
        format: ["MathML", "TeX", "AsciiMath"], // determines type of pre-processors to run
        tex: {}, // configuration options for tex pre-processor
        ascii: {}, // configuration options for ascii pre-processor
        singleDollars: false, // allow single-dollar delimiter for inline TeX
        fragment: false,
        jsdom: {
            FetchExternalResources: [],
            ProcessExternalResources: false,
            // NOTE virtualConsole: not the jsdom way but for configurability (cf. below)
            virtualConsole: true
        },
        //
        // standard mathjax-node options
        //
        displayMessages: false, // determines whether Message.Set() calls are logged
        displayErrors: true, // determines whether error messages are shown on the console
        undefinedCharError: false, // determines whether unknown characters are saved in the error array
        extensions: '', // a convenience option to add MathJax extensions
        fontURL: 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js/fonts/HTML-CSS', // for webfont urls in the CSS for HTML output
        MathJax: {} // options MathJax configuration, see https://docs.mathjax.org
    }
    const mjstate = {};
    // defaults for mathjax-node's typeset method
    const typesetConfig = {
        ex: 6, // ex-size in pixels
        width: 100, // width of container (in ex) for linebreaking and tags
        useFontCache: true, // use <defs> and <use> in svg output?
        useGlobalCache: false, // use common <defs> for all equations?
        state: mjstate, // track global state
        linebreaks: false, // do linebreaking?
        equationNumbers: "none", // or "AMS" or "all"
        math: "", // the math to typeset
        html: false, // generate HTML output?
        css: false, // generate CSS for HTML output?
        mml: false, // generate mml output?
        svg: false, // generate svg output?
        speakText: true, // add spoken annotations to svg output?
        speakRuleset: "mathspeak", // set speech ruleset (default (chromevox rules), mathspeak)
        speakStyle: "default", // set speech style (mathspeak:  default, brief, sbrief)
        timeout: 10 * 1000, // 10 second timeout before restarting MathJax
    }


    //merge configurations

    const mergeConfig = function(conf, confDefaults) {
        for (index in conf) {
            confDefaults[index] = conf[index];
        }
    };

    mergeConfig(configOptions, config);
    mergeConfig(typesetOptions, typesetConfig);

    // Set jsdom console (cf. defaults for config.jsdom)
    if (config.jsdom.virtualConsole) {
        config.jsdom.virtualConsole = jsdom.createVirtualConsole().sendTo(console);
    }

    // set up DOM basics
    const doc = jsdom.jsdom(htmlstring, config.jsdom);
    const window = doc.defaultView;
    const document = window.document;

    // configure mathjax-node
    mathjax.config(config);

    // configure and pre-process
    if (config.format.indexOf('MathML') > -1) {
        window.mathml = mathml;
        window.mathml.config.doc = document;
        window.mathml.PreProcess();
    }
    // HACK clone configuration cf #22
    tex.config = JSON.parse(texconfig);
    if (config.format.indexOf('TeX') > -1) {
        if (config.MathJax.tex2jax) {
            mergeConfig(config.MathJax.tex2jax, tex.config);
        }
        window.tex = tex;
        window.tex.config.doc = document;
        if (config.singleDollars) {
            window.tex.config.inlineMath.push(['$', '$']);
            window.tex.config.processEscapes = true;
        }
        window.tex.PreProcess();
    }
    // HACK clone configuration cf #22
    ascii.config = JSON.parse(asciiconfig);
    if (config.format.indexOf('AsciiMath') > -1) {
        if (config.MathJax.ascii2jax) {
            mergeConfig(config.MathJax.ascii2jax, ascii.config);
        }
        window.ascii = ascii;
        window.ascii.config.doc = document;
        window.ascii.PreProcess();
    }
    //normalize type strings
    const scripts = document.querySelectorAll('script[type^="math/"]');
    for (let i = 0; i < scripts.length; i++) {
        const script = scripts[i];
        let format = script.getAttribute("type").toLowerCase().slice(5);
        if (format === "tex" || format === "inline-tex") {
            format = "math/inline-TeX";
            // TODO Should use match to handle arbitrary spaces that are supported by MathJax
            // https://github.com/mathjax/MathJax/blob/master/unpacked/jax/input/TeX/jax.js#L2168
        } else if (format === "tex; mode=display") {
            format = "math/TeX";
{'fraction_non_alphanumeric': 0.12004456410732522, 'fraction_numerical': 0.006870299879305542, 'mean_word_length': 2.5260229132569556, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2517331', 'n_tokens_mistral': 3141, 'n_tokens_neox': 3014, 'n_words': 844}