<commit_before>///
/// ANSI C grammar
///
/// Based on Jutta Degener's YACC grammar
/// <http://www.lysator.liu.se/c/ANSI-C-grammar-y.html>
///
/// Converted to TameParse format by <NAME>
///

language Ansi-C {
	///
	/// Character types
	///
	lexer-symbols {
		comment-start 		= "/*"
		comment-end			= "*/"
		cpp-comment-start	= "//"

		line-separator		= /[\n\r]/
		whitespace			= /({line-separator}|[ \t])/

		digit				= /[0-9]/
		letter				= /[a-zA-Z_]/
		alphanumeric		= /({digit}|{letter})/
		hex-digit			= /[0-9A-Fa-f]/
		exponent			= /[Ee][+\-]?{digit}+/
		float-specifier		= /[fFlL]/
		int-specifier		= /[uUlL]+/
	}

	///
	/// ANSI C keywords
	///
	keywords {
		auto		break		case		char 		const		continue
		default		do 			double 		else 		enum 		extern 
		float 		for 		goto 		if 			int 		long 
		register 	return 		short 		signed 		sizeof 		static 
		struct 		switch 		typedef 	union 		unsigned 	void 
		volatile 	while
	}

	///
	/// Ignored sections: comments and whitespace
	///
	ignore {
		comment 			= 	/{comment-start}([^*]|\*[^\/]){comment-end}/
		cpp-comment			= 	/{cpp-comment-start}[^\n\r]*/
		whitespace			= 	/{whitespace}+/
		doc-comment			= 	/{comment-start}\*([^*]|*[^\/]){comment-end}/
		doc-comment			|=	/{cpp-comment-start}\/[^\n\r]*/
	}

	///
	/// Lexical symbols
	///
	/// We define some extra lexical symbols for the various types of C constant over
	/// Jutta Deneger's version to make it easier to decide how to interpret these
	/// values later on.
	///
	lexer {
		identifier			= 	/{letter}{alphanumeric}*/
		constant-hex		= 	/0[xX]{hex-digit}+{int-specifier}?/
		constant-octal		= 	/0{digit}+{int-specifier}?/
		constant-decimal	= 	/{digit}+{int-specifier}?/
		constant-character	= 	/L?'(\\.|[^\\'])+'/
		constant-float		= 	/{digit}+{exponent}{float-specifier}?/
		constant-float		|= 	/{digit}*\.{digit}+{exponent}?{float-specifier}?/
		constant-float		|=	/{digit}+\.{digit}*{exponent}?{float-specifier}?/
		string-literal		=	/L?\"(\\.|[^\\\"])*\"/
	}

	grammar {
		<Constant>						= constant-hex
										| constant-octal
										| constant-decimal
										| constant-float
										| constant-character
										| string-literal
	
		<Primary-Expression>			= identifier
										| <Constant>
										| '(' <Expression> ')'
	
		<Postfix-Expression>			= <Primary-Expression>
										| <Postfix-Expression> '[' <Expression> ']'
										| <Postfix-Expression> '(' <Argument-Expression-List>? ')'
										| <Postfix-Expression> '.' identifier
										| <Postfix-Expression> "->" identifier
										| <Postfix-Expression> "++"
										| <Postfix-Expression> "--"
	
		<Argument-Expression-List>		= <Assignment-Expression> (',' <Assignment-Expression>)*	
	
		<Unary-Expression>				= <Postfix-Expression>
										| "++" <Unary-Expression>
										| "--" <Unary-Expression>
										| ('&' | '*' | '+' | '-' | '~' | '!') <Cast-Expression>
										| sizeof <Unary-Expression>
										| sizeof '(' <Type-Name> ')'
	
		//
		// Casts are interesting as they are ambiguous with function calls in a few situations
		//
		// Specifically: (x)(y) can be a cast of y to x or a call to function variable x
		// depending on if x was previously declared as a type or a variable.
		//
		// We would like to produce several cases for this, so that the parser does not need
		// to know how things are declared to proceed.
		//
		// Unambiguous cast: '( <Type-Name> ')' <Not-A-Parameter-List>
		// Ambiguous cast: '(' identifier ')' <Maybe-A-Parameter-List>
		// Function call: '(' identifier ')' '(' <x> ',' <y> ')'
		//
		<Cast-Expression>				= <Unary-Expression>
										//| '(' <Type-Name> ')' <Cast-Expression>
		
		<Ambiguous-Cast-Or-Function>	= '(' identifier [=> <Ambiguous-Cast-Lookahead>] ')' <Cast-Expression>

		<Unambiguous-Cast-Lookahead>	= ')' <Unary-Expression>

		<Ambiguous-Cast-Lookahead>		= ')' '(' <Unary-Expression> ')'
			
		<Multiplicative-Expression>		= <Cast-Expression>
										| <Multiplicative-Expression> ('*' | '/' | '%') <Cast-Expression>
			
		<Additive-Expression>			= <Multiplicative-Expression>
										| <Additive-Expression> ('+' | '-') <Multiplicative-Expression><commit_msg>Cast expressions should work now<commit_after>///
/// ANSI C grammar
///
/// Based on Jutta Degener's YACC grammar
/// <http://www.lysator.liu.se/c/ANSI-C-grammar-y.html>
///
/// Converted to TameParse format by <NAME>
///

language Ansi-C {
	///
	/// Character types
	///
	lexer-symbols {
		comment-start 		= "/*"
		comment-end			= "*/"
		cpp-comment-start	= "//"

		line-separator		= /[\n\r]/
		whitespace			= /({line-separator}|[ \t])/

		digit				= /[0-9]/
		letter				= /[a-zA-Z_]/
		alphanumeric		= /({digit}|{letter})/
		hex-digit			= /[0-9A-Fa-f]/
		exponent			= /[Ee][+\-]?{digit}+/
		float-specifier		= /[fFlL]/
		int-specifier		= /[uUlL]+/
	}

	///
	/// ANSI C keywords
	///
	keywords {
		auto		break		case		char 		const		continue
		default		do 			double 		else 		enum 		extern 
		float 		for 		goto 		if 			int 		long 
		register 	return 		short 		signed 		sizeof 		static 
		struct 		switch 		typedef 	union 		unsigned 	void 
		volatile 	while
	}

	///
	/// Ignored sections: comments and whitespace
	///
	ignore {
		comment 			= 	/{comment-start}([^*]|\*[^\/]){comment-end}/
		cpp-comment			= 	/{cpp-comment-start}[^\n\r]*/
		whitespace			= 	/{whitespace}+/
		doc-comment			= 	/{comment-start}\*([^*]|*[^\/]){comment-end}/
		doc-comment			|=	/{cpp-comment-start}\/[^\n\r]*/
	}

	///
	/// Lexical symbols
	///
	/// We define some extra lexical symbols for the various types of C constant over
	/// Jutta Deneger's version to make it easier to decide how to interpret these
	/// values later on.
	///
	lexer {
		identifier			= 	/{letter}{alphanumeric}*/
		constant-hex		= 	/0[xX]{hex-digit}+{int-specifier}?/
		constant-octal		= 	/0{digit}+{int-specifier}?/
		constant-decimal	= 	/{digit}+{int-specifier}?/
		constant-character	= 	/L?'(\\.|[^\\'])+'/
		constant-float		= 	/{digit}+{exponent}{float-specifier}?/
		constant-float		|= 	/{digit}*\.{digit}+{exponent}?{float-specifier}?/
		constant-float		|=	/{digit}+\.{digit}*{exponent}?{float-specifier}?/
		string-literal		=	/L?\"(\\.|[^\\\"])*\"/
	}

	grammar {
		<Constant>						= constant-hex
										| constant-octal
										| constant-decimal
										| constant-float
										| constant-character
										| string-literal
	
		<Primary-Expression>			= identifier
										| <Constant>
										| '(' <Expression> ')'
	
		<Postfix-Expression>			= <Primary-Expression>
										| <Postfix-Expression> '[' <Expression> ']'
										| <Postfix-Expression> '(' <Argument-Expression-List>? ')'
										| <Postfix-Expression> '.' identifier
										| <Postfix-Expression> "->" identifier
										| <Postfix-Expression> "++"
										| <Postfix-Expression> "--"
	
		<Argument-Expression-List>		= <Assignment-Expression> (',' <Assignment-Expression>)*	
	
		<Unary-Expression>				= <Postfix-Expression>
										| "++" <Unary-Expression>
										| "--" <Unary-Expression>
										| ('&' | '*' | '+' | '-' | '~' | '!') <Cast-Expression>
										| sizeof <Unary-Expression>
										| sizeof '(' <Type-Name> ')'
	
		//
		// Casts are interesting as they are ambiguous with function calls in a few situations
		//
		// Specifically: (x)(y) can be a cast of y to x or a call to function variable x
		// depending on if x was previously declared as a type or a variable.
		//
		// We would like to produce several cases for this, so that the parser does not need
		// to know how things are declared to proceed.
		//
		// Unambiguous cast: '( <Type-Name> ')' <Not-A-Parameter-List>
		// Ambiguous cast: '(' identifier ')' <Maybe-A-Parameter-List>
		// Multi-level ambiguous cast: '(' identifier ')' '(' identifier ')' '(' ... ')'
		// Function call: '(' identifier ')' '(' <x> ',' <y> ')'
		//
		// We resolve the ambiguous cases as casts for now.
		//
		<Cast-Expression>				= <Unary-Expression>
										| '(' <Type-Name> [=> ')' <Cast-Expression>] ')' <Cast-Expression>
			
		<Multiplicative-Expression>		= <Cast-Expression>
										| <Multiplicative-Expression> ('*' | '/' | '%') <Cast-Expression>
			
		<Additive-Expression>			= <Multiplicative-Expression>
										| <Additive-Expression> ('+' | '-') <Multiplicative-Expression>
{'fraction_non_alphanumeric': 0.19446458786560308, 'fraction_numerical': 0.0014503263234227702, 'mean_word_length': 2.3073541167066347, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1954404', 'n_tokens_mistral': 3700, 'n_tokens_neox': 2977, 'n_words': 578}