<commit_before>/*==================================================
 *  Exhibit.Expression
 *  http://simile.mit.edu/wiki/Exhibit/API/Expression
 *==================================================
 */
Exhibit.Expression = new Object();

Exhibit.Expression.parse = function(s) {
    var expressions = Exhibit.Expression.parseSeveral(s);
    if (expressions.length > 1) {
        throw new Error("Expected only one expression");
    }
    
    return expressions[0];
};

Exhibit.Expression.parseSeveral = function(s) {
    var tokens = Exhibit.Expression._tokenize(s);
    
    var x = -1;
    var token;
    var consumeToken = function() {
        ++x;
        token = x < tokens.length ? tokens[x] : null;
    };
    consumeToken();
    
    var parseFactor = function() {
        if (token == null) {
            throw new Error("Missing factor");
        }
        
        var result = null;
        
        switch (token.type) {
        case Exhibit.Expression._Token.NUMBER:
            result = new Exhibit.Expression._Constant(token.value, "number");
            break;
        case Exhibit.Expression._Token.STRING:
            result = new Exhibit.Expression._Constant(token.value, "text");
            break;
        case Exhibit.Expression._Token.PATH:
            result = token.value;
            break;
        case Exhibit.Expression._Token.FUNCTION:
            var name = token.value;
            consumeToken();
            
            if (token.type == Exhibit.Expression._Token.DELIMITER && token.value == "(") {
                consumeToken();
                
                result = new Exhibit.Expression._FunctionCall(name, parseExpressionList());
                
                if (token == null || token.type != Exhibit.Expression._Token.DELIMITER || token.value != ")") {
                    throw new Error("Missing ) after function call");
                }
            } else {
                throw new Error("Missing ( after function name");
            }
            break;
        case Exhibit.Expression._Token.CONTROL:
            var name = token.value;
            consumeToken();
            
            if (token.type == Exhibit.Expression._Token.DELIMITER && token.value == "(") {
                consumeToken();
                
                result = new Exhibit.Expression._ControlCall(name, parseExpressionList());
                
                if (token == null || token.type != Exhibit.Expression._Token.DELIMITER || token.value != ")") {
                    throw new Error("Missing ) to end " + name);
                }
            } else {
                throw new Error("Missing ( to start " + name);
            }
            break;
        case Exhibit.Expression._Token.DELIMITER:
            if (token.value == "(") {
                consumeToken();
                
                result = parseExpression();
                if (token == null || token.type != Exhibit.Expression._Token.DELIMITER || token.value != ")") {
                    throw new Error("Missing )");
                }
                break;
            } // else, fall through
        default:
            throw new Error("Unexpected token " + token);
        }
        
        consumeToken();
        return result;
    };
    var parseTerm = function() {
        var term = parseFactor();
        while (token != null && 
            token.type == Exhibit.Expression._Token.OPERATOR && 
            (token.value == "*" || token.value == "/")) {
            var operator = token.value;
            consumeToken();
            
            term = new Exhibit.Expression._Operator(operator, [ term, parseFactor() ]);
        }
        return term;
    };
    var parseSubExpression = function() {
        var subExpression = parseTerm();
        while (token != null && 
            token.type == Exhibit.Expression._Token.OPERATOR && 
            (token.value == "+" || token.value == "-")) {
            var operator = token.value;
            consumeToken();
            
            subExpression = new Exhibit.Expression._Operator(operator, [ subExpression, parseTerm() ]);
        }
        return subExpression;
    };
    var parseExpression = function() {
        var expression = parseSubExpression();
        while (token != null && 
            token.type == Exhibit.Expression._Token.OPERATOR && 
            (token.value == "=" || 
             token.value == "!=" || 
             token.value == "<" || 
             token.value == "<=" || 
             token.value == ">" || 
             token.value == ">="
            )) {
            var operator = token.value;
            consumeToken();
            
            expression = new Exhibit.Expression._Operator(operator, [ expression, parseSubExpression() ]);
        }
        return expression;
    };
    var parseExpressionList = function() {
        var expressions = [ parseExpression() ];
        while (token != null && token.type == Exhibit.Expression._Token.DELIMITER && token.value == ",") {
            consumeToken();
            
            expressions.push(parseExpression());
        }
        return expressions;
    }
    
    var roots = parseExpressionList();
    if (token != null) {
        throw new Error("Extra text found at the end of the code");
    }
    
    var expressions = [];
    for (var r = 0; r < roots.length; r++) {
        expressions.push(new Exhibit.Expression._Impl(roots[r]));
    }
    
    return expressions;
};

Exhibit.Expression._delimiters = { "(":true, ")":true, ",":true };
Exhibit.Expression._tokenizeNoStringLiterals = function(s) {
    s = s.replace(/\s+/g, ' ').
        replace(/\s+\./g, '.').
        replace(/\s+\!/g, '!').
        replace(/\s?\(\s?/g, ' ( ').
        replace(/\s?\)\s?/g, ' ) ').
        replace(/\s\+\s/g, ' + ').
        replace(/\s\-\s/g, ' - ').
        replace(/\s\*\s/g, ' * ').
        replace(/\s\/\s/g, ' / ').
        replace(/\s?\,\s?/g, ' , ').
        trim();
        
    var tokens = s.split(" ");
    var results = [];
    for (var i = 0; i < tokens.length; i++) {
        var token = tokens[i].trim();
        if (token == "") {
            // continue;
        } else if (token in Exhibit.Expression._delimiters) {
            results.push(new Exhibit.Expression._Token(Exhibit.Expression._Token.DELIMITER, token));
        } else if (token in Exhibit.Expression._operators) {
            results.push(new Exhibit.Expression._Token(Exhibit.Expression._Token.OPERATOR, token));
        } else if (token in Exhibit.Functions) {
            results.push(new Exhibit.Expression._Token(Exhibit.Expression._Token.FUNCTION, token));<commit_msg>Temporarily fixed expression parser to handle +, -, *, / operators.<commit_after>/*==================================================
 *  Exhibit.Expression
 *  http://simile.mit.edu/wiki/Exhibit/API/Expression
 *==================================================
 */
Exhibit.Expression = new Object();

Exhibit.Expression.parse = function(s) {
    var expressions = Exhibit.Expression.parseSeveral(s);
    if (expressions.length > 1) {
        throw new Error("Expected only one expression");
    }
    
    return expressions[0];
};

Exhibit.Expression.parseSeveral = function(s) {
    var tokens = Exhibit.Expression._tokenize(s);
    
    var x = -1;
    var token;
    var consumeToken = function() {
        ++x;
        token = x < tokens.length ? tokens[x] : null;
    };
    consumeToken();
    
    var parseFactor = function() {
        if (token == null) {
            throw new Error("Missing factor");
        }
        
        var result = null;
        
        switch (token.type) {
        case Exhibit.Expression._Token.NUMBER:
            result = new Exhibit.Expression._Constant(token.value, "number");
            break;
        case Exhibit.Expression._Token.STRING:
            result = new Exhibit.Expression._Constant(token.value, "text");
            break;
        case Exhibit.Expression._Token.PATH:
            result = token.value;
            break;
        case Exhibit.Expression._Token.FUNCTION:
            var name = token.value;
            consumeToken();
            
            if (token.type == Exhibit.Expression._Token.DELIMITER && token.value == "(") {
                consumeToken();
                
                result = new Exhibit.Expression._FunctionCall(name, parseExpressionList());
                
                if (token == null || token.type != Exhibit.Expression._Token.DELIMITER || token.value != ")") {
                    throw new Error("Missing ) after function call");
                }
            } else {
                throw new Error("Missing ( after function name");
            }
            break;
        case Exhibit.Expression._Token.CONTROL:
            var name = token.value;
            consumeToken();
            
            if (token.type == Exhibit.Expression._Token.DELIMITER && token.value == "(") {
                consumeToken();
                
                result = new Exhibit.Expression._ControlCall(name, parseExpressionList());
                
                if (token == null || token.type != Exhibit.Expression._Token.DELIMITER || token.value != ")") {
                    throw new Error("Missing ) to end " + name);
                }
            } else {
                throw new Error("Missing ( to start " + name);
            }
            break;
        case Exhibit.Expression._Token.DELIMITER:
            if (token.value == "(") {
                consumeToken();
                
                result = parseExpression();
                if (token == null || token.type != Exhibit.Expression._Token.DELIMITER || token.value != ")") {
                    throw new Error("Missing )");
                }
                break;
            } // else, fall through
        default:
            throw new Error("Unexpected token " + token);
        }
        
        consumeToken();
        return result;
    };
    var parseTerm = function() {
        var term = parseFactor();
        while (token != null && 
            token.type == Exhibit.Expression._Token.OPERATOR && 
            (token.value == "*" || token.value == "/")) {
            var operator = token.value;
            consumeToken();
            
            term = new Exhibit.Expression._Operator(operator, [ term, parseFactor() ]);
        }
        return term;
    };
    var parseSubExpression = function() {
        var subExpression = parseTerm();
        while (token != null && 
            token.type == Exhibit.Expression._Token.OPERATOR && 
            (token.value == "+" || token.value == "-")) {
            var operator = token.value;
            consumeToken();
            
            subExpression = new Exhibit.Expression._Operator(operator, [ subExpression, parseTerm() ]);
        }
        return subExpression;
    };
    var parseExpression = function() {
        var expression = parseSubExpression();
        while (token != null && 
            token.type == Exhibit.Expression._Token.OPERATOR && 
            (token.value == "=" || 
             token.value == "!=" || 
             token.value == "<" || 
             token.value == "<=" || 
             token.value == ">" || 
             token.value == ">=" ||
             token.value == "+" ||
             token.value == "-" ||
             token.value == "*" ||
             token.value == "/"
            )) {
            var operator = token.value;
            consumeToken();
            
            expression = new Exhibit.Expression._Operator(operator, [ expression, parseSubExpression() ]);
        }
        return expression;
    };
    var parseExpressionList = function() {
        var expressions = [ parseExpression() ];
        while (token != null && token.type == Exhibit.Expression._Token.DELIMITER && token.value == ",") {
            consumeToken();
            
            expressions.push(parseExpression());
        }
        return expressions;
    }
    
    var roots = parseExpressionList();
    if (token != null) {
        throw new Error("Extra text found at the end of the code");
    }
    
    var expressions = [];
    for (var r = 0; r < roots.length; r++) {
        expressions.push(new Exhibit.Expression._Impl(roots[r]));
    }
    
    return expressions;
};

Exhibit.Expression._delimiters = { "(":true, ")":true, ",":true };
Exhibit.Expression._tokenizeNoStringLiterals = function(s) {
    s = s.replace(/\s+/g, ' ').
        replace(/\s?\(\s?/g, ' ( ').
        replace(/\s?\)\s?/g, ' ) ').
        replace(/\s\+\s/g, ' + ').
        replace(/\s\-\s/g, ' - ').
        replace(/\s\*\s/g, ' * ').
        replace(/\s\/\s/g, ' / ').
        replace(/\s?\,\s?/g, ' , ').
        trim();
        
    var tokens = s.split(" ");
    var results = [];
    for (var i = 0; i < tokens.length; i++) {
        var token = tokens[i].trim();
        if (token == "") {
            // continue;
        } else if (token in Exhibit.Expression._delimiters) {
            results.push(new Exhibit.Expression._Token(Exhibit.Expression._Token.DELIMITER, token));
        } else if (token in Exhibit.Expression._operators) {
            results.push(new Exhibit.Expression._Token(Exhibit.Expression._Token.OPERATOR, token));
        } else if (token in Exhibit.Functions) {
            results.push(new Exhibit.Expression._Token(Exhibit.Expression._Token.FUNCTION, token));
{'fraction_non_alphanumeric': 0.15452242586797796, 'fraction_numerical': 0.0007450454477723141, 'mean_word_length': 1.6728395061728396, 'pattern_counts': {'":': 6, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2121341', 'n_tokens_mistral': 3572, 'n_tokens_neox': 3425, 'n_words': 721}