<commit_before>var getExpression;

// expression
(function () {
	var getExpressionList,
	makePrefixSequenceMatcher,
	makeInfixSequenceMatcher,
	getBracketedExpression,
	getPrimary,
	getMemberOrInvocation,
	getTypeOf,
	getLogicalOr,
	getConditional,
	
	getDigits,
	getExponent,
	getFraction,
	getInteger,
	
	getReference,
	getRefinement,

	getLiteral,
	getArrayLiteral,
	getBooleanLiteral,
	getNumberLiteral,
	getStringLiteral,
	getObjectLiteral,

	getKeyValuePairs,
	getKeyValuePair,
	getKey,

	getName,

	getDotRefinement,
	getArrayRefinement,
	getArrayMember,

	globals;

	getExpression = function ( tokenizer ) {
		// The conditional operator is the lowest precedence operator (except yield,
		// assignment operators, and commas, none of which are supported), so we
		// start there. If it doesn't match, it 'falls through' to progressively
		// higher precedence operators, until it eventually matches (or fails to
		// match) a 'primary' - a literal or a reference. This way, the abstract syntax
		// tree has everything in its proper place, i.e. 2 + 3 * 4 === 14, not 20.
		return getConditional( tokenizer );
	};

	getExpressionList = function ( tokenizer ) {
		var start, expressions, expr, next;

		start = tokenizer.pos;

		allowWhitespace( tokenizer );

		expr = getExpression( tokenizer );

		if ( expr === null ) {
			return null;
		}

		expressions = [ expr ];

		// allow whitespace between expression and ','
		allowWhitespace( tokenizer );

		if ( getStringMatch( tokenizer, ',' ) ) {
			next = getExpressionList( tokenizer );
			if ( next === null ) {
				tokenizer.pos = start;
				return null;
			}

			expressions = expressions.concat( next );
		}

		return expressions;
	};

	getBracketedExpression = function ( tokenizer ) {
		var start, expr;

		start = tokenizer.pos;

		if ( !getStringMatch( tokenizer, '(' ) ) {
			return null;
		}

		allowWhitespace( tokenizer );

		expr = getExpression( tokenizer );
		if ( !expr ) {
			tokenizer.pos = start;
			return null;
		}

		allowWhitespace( tokenizer );

		if ( !getStringMatch( tokenizer, ')' ) ) {
			tokenizer.pos = start;
			return null;
		}

		return {
			t: BRACKETED,
			x: expr
		};
	};

	getPrimary = function ( tokenizer ) {
		return getLiteral( tokenizer )
		    || getReference( tokenizer )
		    || getBracketedExpression( tokenizer );
	};

	getMemberOrInvocation = function ( tokenizer ) {
		var current, expression, refinement, expressionList;

		expression = getPrimary( tokenizer );

		if ( !expression ) {
			return null;
		}

		while ( expression ) {
			current = tokenizer.pos;

			if ( refinement = getRefinement( tokenizer ) ) {
				expression = {
					t: MEMBER,
					x: expression,
					r: refinement
				};
			}

			else if ( getStringMatch( tokenizer, '(' ) ) {
				allowWhitespace( tokenizer );
				expressionList = getExpressionList( tokenizer );

				allowWhitespace( tokenizer );

				if ( !getStringMatch( tokenizer, ')' ) ) {
					tokenizer.pos = current;
					break;
				}

				expression = {
					t: INVOCATION,
					x: expression
				};

				if ( expressionList ) {
					expression.o = expressionList;
				}
			}

			else {
				break;
			}
		}

		return expression;
	};

	// right-to-left
	makePrefixSequenceMatcher = function ( symbol, fallthrough ) {
		return function ( tokenizer ) {
			var start, expression;

			if ( !getStringMatch( tokenizer, symbol ) ) {
				return fallthrough( tokenizer );
			}

			start = tokenizer.pos;

			allowWhitespace( tokenizer );

			expression = getExpression( tokenizer );
			if ( !expression ) {
				fail( tokenizer, 'an expression' );
			}

			return {
				s: symbol,
				o: expression,
				t: PREFIX_OPERATOR
			};
		};
	};

	// create all prefix sequence matchers
	(function () {
		var i, len, matcher, prefixOperators, fallthrough;

		prefixOperators = '! ~ + - typeof'.split( ' ' );

		// An invocation refinement is higher precedence than logical-not
		//fallthrough = getInvocationRefinement;
		fallthrough = getMemberOrInvocation;
		for ( i=0, len=prefixOperators.length; i<len; i+=1 ) {
			matcher = makePrefixSequenceMatcher( prefixOperators[i], fallthrough );
			fallthrough = matcher;
		}

		// typeof operator is higher precedence than multiplication, so provides the
		// fallthrough for the multiplication sequence matcher we're about to create
		// (we're skipping void and delete)
		getTypeOf = fallthrough;
	}());<commit_msg>fix prefix operator precedence in expression AST<commit_after>var getExpression;

// expression
(function () {
	var getExpressionList,
	makePrefixSequenceMatcher,
	makeInfixSequenceMatcher,
	getBracketedExpression,
	getPrimary,
	getMemberOrInvocation,
	getTypeOf,
	getLogicalOr,
	getConditional,
	
	getDigits,
	getExponent,
	getFraction,
	getInteger,
	
	getReference,
	getRefinement,

	getLiteral,
	getArrayLiteral,
	getBooleanLiteral,
	getNumberLiteral,
	getStringLiteral,
	getObjectLiteral,

	getKeyValuePairs,
	getKeyValuePair,
	getKey,

	getName,

	getDotRefinement,
	getArrayRefinement,
	getArrayMember,

	globals;

	getExpression = function ( tokenizer ) {
		// The conditional operator is the lowest precedence operator (except yield,
		// assignment operators, and commas, none of which are supported), so we
		// start there. If it doesn't match, it 'falls through' to progressively
		// higher precedence operators, until it eventually matches (or fails to
		// match) a 'primary' - a literal or a reference. This way, the abstract syntax
		// tree has everything in its proper place, i.e. 2 + 3 * 4 === 14, not 20.
		return getConditional( tokenizer );
	};

	getExpressionList = function ( tokenizer ) {
		var start, expressions, expr, next;

		start = tokenizer.pos;

		allowWhitespace( tokenizer );

		expr = getExpression( tokenizer );

		if ( expr === null ) {
			return null;
		}

		expressions = [ expr ];

		// allow whitespace between expression and ','
		allowWhitespace( tokenizer );

		if ( getStringMatch( tokenizer, ',' ) ) {
			next = getExpressionList( tokenizer );
			if ( next === null ) {
				tokenizer.pos = start;
				return null;
			}

			expressions = expressions.concat( next );
		}

		return expressions;
	};

	getBracketedExpression = function ( tokenizer ) {
		var start, expr;

		start = tokenizer.pos;

		if ( !getStringMatch( tokenizer, '(' ) ) {
			return null;
		}

		allowWhitespace( tokenizer );

		expr = getExpression( tokenizer );
		if ( !expr ) {
			tokenizer.pos = start;
			return null;
		}

		allowWhitespace( tokenizer );

		if ( !getStringMatch( tokenizer, ')' ) ) {
			tokenizer.pos = start;
			return null;
		}

		return {
			t: BRACKETED,
			x: expr
		};
	};

	getPrimary = function ( tokenizer ) {
		return getLiteral( tokenizer )
		    || getReference( tokenizer )
		    || getBracketedExpression( tokenizer );
	};

	getMemberOrInvocation = function ( tokenizer ) {
		var current, expression, refinement, expressionList;

		expression = getPrimary( tokenizer );

		if ( !expression ) {
			return null;
		}

		while ( expression ) {
			current = tokenizer.pos;

			if ( refinement = getRefinement( tokenizer ) ) {
				expression = {
					t: MEMBER,
					x: expression,
					r: refinement
				};
			}

			else if ( getStringMatch( tokenizer, '(' ) ) {
				allowWhitespace( tokenizer );
				expressionList = getExpressionList( tokenizer );

				allowWhitespace( tokenizer );

				if ( !getStringMatch( tokenizer, ')' ) ) {
					tokenizer.pos = current;
					break;
				}

				expression = {
					t: INVOCATION,
					x: expression
				};

				if ( expressionList ) {
					expression.o = expressionList;
				}
			}

			else {
				break;
			}
		}

		return expression;
	};

	// right-to-left
	makePrefixSequenceMatcher = function ( symbol, fallthrough ) {
		return function ( tokenizer ) {
			var start, expression;

			if ( !getStringMatch( tokenizer, symbol ) ) {
				return fallthrough( tokenizer );
			}

			start = tokenizer.pos;

			allowWhitespace( tokenizer );

			expression = getTypeOf( tokenizer );
			if ( !expression ) {
				fail( tokenizer, 'an expression' );
			}

			return {
				s: symbol,
				o: expression,
				t: PREFIX_OPERATOR
			};
		};
	};

	// create all prefix sequence matchers
	(function () {
		var i, len, matcher, prefixOperators, fallthrough;

		prefixOperators = '! ~ + - typeof'.split( ' ' );

		// An invocation refinement is higher precedence than logical-not
		//fallthrough = getInvocationRefinement;
		fallthrough = getMemberOrInvocation;
		for ( i=0, len=prefixOperators.length; i<len; i+=1 ) {
			matcher = makePrefixSequenceMatcher( prefixOperators[i], fallthrough );
			fallthrough = matcher;
		}

		// typeof operator is higher precedence than multiplication, so provides the
		// fallthrough for the multiplication sequence matcher we're about to create
		// (we're skipping void and delete)
		getTypeOf = fallthrough;
	}());
{'fraction_non_alphanumeric': 0.10600385881284757, 'fraction_numerical': 0.0020429009193054137, 'mean_word_length': 3.2062052505966587, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7151941', 'n_tokens_mistral': 3230, 'n_tokens_neox': 2665, 'n_words': 738}