<commit_before>'use strict';

let jsep = require('jsep');

// Only algebraic expressions.
['!', '~'].forEach(jsep.removeUnaryOp);
['||', '&&', '|', '^', '&', '<<', '>>', '>>>'].forEach(jsep.removeBinaryOp);
['==', '!=', '===', '!==', '<', '>', '<=', '>='].forEach(jsep.removeBinaryOp);
['×', '⋅', ' '].forEach(op => jsep.addBinaryOp(op ,10) );
['^', '**'].forEach(op => jsep.addBinaryOp(op, 11) );

let superscriptToDigit = {
    '⁰': 0,
    '¹': 1,
    '²': 2,
    '³': 3,
    '⁴': 4,
    '⁵': 5,
    '⁶': 6,
    '⁷': 7,
    '⁸': 8,
    '⁹': 9,
    '⁻': '-',
    '⁺': '+'
};

let digitToSuperscript = {
    '0': '⁰',
    '1': '¹',
    '2': '²',
    '3': '³',
    '4': '⁴',
    '5': '⁵',
    '6': '⁶',
    '7': '⁷',
    '8': '⁸',
    '9': '⁹',
    '-': '⁻',
    '+': '⁺'
};

function quirks(node) {
    if (node.type === 'BinaryExpression') {
        node.left = quirks(node.left);
        node.right = quirks(node.right);
    }

     // Transform <unary minus> <literal> into <literal>
    if (node.type === 'UnaryExpression' && node.operator === '-' && node.argument.type == 'Literal') {
        node = node.argument;
        node.value = node.value.times(-1);
        node.raw = '-' + node.raw;
    }

    if (node.type === 'UnaryExpression') {
        node.argument = quirks(node.argument);
    }

    return node;
}

function parse(s, options) {
    options = Object.assign({}, {
        implicitMultiplication: true,
        digitGroupSeparator: ' '
    }, options || {});

    // Transform numeric superscripts into '^x'.
    s = s.replace(/[⁻⁺]?[⁰¹²³⁴⁵⁶⁷⁸⁹]+/g, (match) => {
        return '^' + Array.from(match, c => superscriptToDigit[c]).join('');
    });

    let ast = jsep(s, options);

    return quirks(ast);
}

let unaryFns = {
    '+': a => a,
    '-': a => a.times(-1)
};

let binaryFns = {
    '+': (a,b) => a.plus(b),
    '-': (a,b) => a.minus(b),
    '/': (a,b) => a.dividedBy(b),
    '*': (a,b) => a.times(b),
    '×': (a,b) => a.times(b),
    '⋅': (a,b) => a.times(b),
    '^': (a,b) => a.pow(b.toNumber()),
    '**': (a,b) => a.pow(b.toNumber())
};

function evaluate(node, unaryOps, binaryOps) {
    if (node.type === 'Identifier')
        throw new Error("'" + node.name + "' is undefined");
    if (node.type === 'Literal')
        return node.value;
    if (node.type === 'BinaryExpression') {
        let left = evaluate(node.left, unaryOps, binaryOps),
            right = evaluate(node.right, unaryOps, binaryOps);
        return binaryOps[node.operator](left, right);
    }
    if (node.type === 'UnaryExpression') {
        let a = evaluate(node.argument, unaryOps, binaryOps);
        return unaryOps[node.operator](a);
    }

    throw new Error("Expression type '" + node.type + "' is not known");<commit_msg>Support unary '%'.<commit_after>'use strict';

let jsep = require('jsep');

// Only algebraic expressions.
['!', '~'].forEach(jsep.removeUnaryOp);
['||', '&&', '|', '^', '&', '<<', '>>', '>>>'].forEach(jsep.removeBinaryOp);
['==', '!=', '===', '!==', '<', '>', '<=', '>='].forEach(jsep.removeBinaryOp);
['×', '⋅', ' '].forEach(op => jsep.addBinaryOp(op ,10) );
['^', '**'].forEach(op => jsep.addBinaryOp(op, 11) );

let superscriptToDigit = {
    '⁰': 0,
    '¹': 1,
    '²': 2,
    '³': 3,
    '⁴': 4,
    '⁵': 5,
    '⁶': 6,
    '⁷': 7,
    '⁸': 8,
    '⁹': 9,
    '⁻': '-',
    '⁺': '+'
};

let digitToSuperscript = {
    '0': '⁰',
    '1': '¹',
    '2': '²',
    '3': '³',
    '4': '⁴',
    '5': '⁵',
    '6': '⁶',
    '7': '⁷',
    '8': '⁸',
    '9': '⁹',
    '-': '⁻',
    '+': '⁺'
};

function quirks(node) {
    if (node.type === 'BinaryExpression') {
        node.left = quirks(node.left);
        node.right = quirks(node.right);
    }

     // Transform <unary minus> <literal> into <literal>
    if (node.type === 'UnaryExpression' && node.operator === '-' && node.argument.type == 'Literal') {
        node = node.argument;
        node.value = node.value.times(-1);
        node.raw = '-' + node.raw;
    }

    if (node.type === 'UnaryExpression') {
        node.argument = quirks(node.argument);
    }

    return node;
}

function parse(s, options) {
    options = Object.assign({}, {
        implicitMultiplication: true,
        digitGroupSeparator: ' '
    }, options || {});

    // Transform numeric superscripts into '^x'.
    s = s.replace(/[⁻⁺]?[⁰¹²³⁴⁵⁶⁷⁸⁹]+/g, (match) => {
        return '^' + Array.from(match, c => superscriptToDigit[c]).join('');
    });

    let ast = jsep(s, options);

    return quirks(ast);
}

let unaryFns = {
    '+': a => a,
    '-': a => a.times(-1),
    '%': a => a.times(0.01)
};

let binaryFns = {
    '+': (a,b) => a.plus(b),
    '-': (a,b) => a.minus(b),
    '/': (a,b) => a.dividedBy(b),
    '*': (a,b) => a.times(b),
    '×': (a,b) => a.times(b),
    '⋅': (a,b) => a.times(b),
    '^': (a,b) => a.pow(b.toNumber()),
    '**': (a,b) => a.pow(b.toNumber())
};

function evaluate(node, unaryOps, binaryOps) {
    if (node.type === 'Identifier')
        throw new Error("'" + node.name + "' is undefined");
    if (node.type === 'Literal')
        return node.value;
    if (node.type === 'BinaryExpression') {
        let left = evaluate(node.left, unaryOps, binaryOps),
            right = evaluate(node.right, unaryOps, binaryOps);
        return binaryOps[node.operator](left, right);
    }
    if (node.type === 'UnaryExpression') {
        let a = evaluate(node.argument, unaryOps, binaryOps);
        return unaryOps[node.operator](a);
    }

    throw new Error("Expression type '" + node.type + "' is not known");
{'fraction_non_alphanumeric': 0.2599493487698987, 'fraction_numerical': 0.009949348769898697, 'mean_word_length': 2.557915057915058, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1021657', 'n_tokens_mistral': 2296, 'n_tokens_neox': 2185, 'n_words': 413}