<commit_before>/*
 * Copyright (C) 2007-2013 <NAME>.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include "Format.h"
#include "FloatLiteral.h"
#include "IntegerLiteral.h"
#include "YasonParser.h"

namespace flux
{

YasonParser::YasonParser()
{
	SYNTAX("Yason");

	IMPORT(floatLiteral(), "float");
	IMPORT(integerLiteral(), "int");

	DEFINE_VOID("CommentLine",
		GLUE(
			STRING("//"),
			CHOICE(
				FIND(AHEAD(CHAR('\n'))),
				FIND(EOI())
			)
		)
	);

	DEFINE_VOID("CommentText",
		GLUE(
			STRING("/*"),
			REPEAT(
				CHOICE(
					INLINE("CommentText"),
					GLUE(
						NOT(STRING("*/")),
						ANY()
					)
				)
			),
			EXPECT("missing closing '*/'",
				STRING("*/")
			)
		)
	);

	DEFINE_VOID("Noise",
		REPEAT(
			CHOICE(
				RANGE(" \t\n\r"),
				INLINE("CommentLine"),
				INLINE("CommentText")
			)
		)
	);

	DEFINE_VOID("EscapedChar",
		GLUE(
			CHAR('\\'),
			EXPECT("illegal escape sequence",
				CHOICE(
					RANGE("\"\\/fnrt"),
					GLUE(
						CHAR('u'),
						REPEAT(4, 4,
							CHOICE(
								RANGE('0', '9'),
								RANGE('a', 'f'),
								RANGE('A', 'F')
							)
						)
					)
				)
			)
		)
	);

	item_ =
		DEFINE("Item",
			REPEAT(1,
				GLUE(
					NOT(
						GLUE(
							INLINE("Noise"),
							RANGE(",]")
						)
					),
					EXCEPT(",;{}:\"\n\r")
				)
			)
		);

	line_ =
		DEFINE("Line",
			GLUE(
				NOT(RANGE("[{")),
				LENGTH(1,
					FIND(
						AHEAD(
							GLUE(
								REPEAT(
									CHOICE(
										RANGE(" \t\r"),
										INLINE("CommentLine"),
										INLINE("CommentText")
									)
								),
								CHOICE(
									AHEAD(CHAR('\n')),
									EOI()
								)
							)
						)
					)
				)
			)
		);

	string_ =<commit_msg>Fixed: correctly recognize single line single member objects<commit_after>/*
 * Copyright (C) 2007-2013 <NAME>.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include "Format.h"
#include "FloatLiteral.h"
#include "IntegerLiteral.h"
#include "YasonParser.h"

namespace flux
{

YasonParser::YasonParser()
{
	SYNTAX("Yason");

	IMPORT(floatLiteral(), "float");
	IMPORT(integerLiteral(), "int");

	DEFINE_VOID("CommentLine",
		GLUE(
			STRING("//"),
			CHOICE(
				FIND(AHEAD(CHAR('\n'))),
				FIND(EOI())
			)
		)
	);

	DEFINE_VOID("CommentText",
		GLUE(
			STRING("/*"),
			REPEAT(
				CHOICE(
					INLINE("CommentText"),
					GLUE(
						NOT(STRING("*/")),
						ANY()
					)
				)
			),
			EXPECT("missing closing '*/'",
				STRING("*/")
			)
		)
	);

	DEFINE_VOID("Noise",
		REPEAT(
			CHOICE(
				RANGE(" \t\n\r"),
				INLINE("CommentLine"),
				INLINE("CommentText")
			)
		)
	);

	DEFINE_VOID("EscapedChar",
		GLUE(
			CHAR('\\'),
			EXPECT("illegal escape sequence",
				CHOICE(
					RANGE("\"\\/fnrt"),
					GLUE(
						CHAR('u'),
						REPEAT(4, 4,
							CHOICE(
								RANGE('0', '9'),
								RANGE('a', 'f'),
								RANGE('A', 'F')
							)
						)
					)
				)
			)
		)
	);

	item_ =
		DEFINE("Item",
			REPEAT(1,
				GLUE(
					NOT(
						GLUE(
							INLINE("Noise"),
							RANGE(",]")
						)
					),
					EXCEPT(",;{}:\"\n\r")
				)
			)
		);

	line_ =
		DEFINE("Line",
			GLUE(
				NOT(RANGE("[{")),
				LENGTH(1,
					FIND(
						AHEAD(
							GLUE(
								REPEAT(
									CHOICE(
										RANGE(" \t\r}"),
										INLINE("CommentLine"),
										INLINE("CommentText")
									)
								),
								CHOICE(
									AHEAD(CHAR('\n')),
									EOI()
								)
							)
						)
					)
				)
			)
		);

	string_ =
{'fraction_non_alphanumeric': 0.18205394190871368, 'fraction_numerical': 0.007780082987551867, 'mean_word_length': 1.948776758409786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2327535', 'n_tokens_mistral': 2122, 'n_tokens_neox': 1431, 'n_words': 273}