<commit_before>//
//  language_parser.cpp
//  TameParse
//
//  Created by <NAME> on 19/09/2011.
//  Copyright 2011 <NAME>. All rights reserved.
//

#include <sstream>

#include "TameParse/Util/stringreader.h"
#include "TameParse/Util/utf8reader.h"
#include "TameParse/Language/language_parser.h"
#include "TameParse/Language/process.h"

#include "tameparse_language.h"

using namespace std;
using namespace util;
using namespace dfa;
using namespace compiler;
using namespace language;

/// \brief Creates a new language object
language_parser::language_parser()
: m_FileDefinition(NULL, true) {
}

typedef tameparse_language::Parser_Language_n                   Parser_Language;
typedef tameparse_language::list_of_TopLevel_Block_n            list_of_TopLevel_Block;
typedef tameparse_language::TopLevel_Block_n                    ast_TopLevel_Block;
typedef tameparse_language::Language_Block_n                    ast_Language_Block;
typedef tameparse_language::Language_Definition_n               ast_Language_Definition;
typedef tameparse_language::list_of_Language_Definition_n       list_of_Language_Definition;
typedef tameparse_language::Lexeme_Definition_n                 ast_Lexeme_Definition;
typedef tameparse_language::list_of_Lexeme_Definition_n         list_of_Lexeme_Definition;
typedef tameparse_language::list_of_Keyword_Definition_n        list_of_Keyword_Definition;
typedef tameparse_language::identifier_n                        identifier;
typedef tameparse_language::regex_n                             ast_regex;
typedef tameparse_language::string_2_n                          ast_string;
typedef tameparse_language::character_n                         ast_character;
typedef tameparse_language::list_of_Nonterminal_Definition_n    list_of_Nonterminal_Definition;
typedef tameparse_language::list_of__pipe__Production_n         list_of__pipe__Production;
typedef tameparse_language::list_of_Simple_Ebnf_Item_n          list_of_Simple_Ebnf_Item;
typedef tameparse_language::Nonterminal_Definition_n            ast_Nonterminal_Definition;
typedef tameparse_language::Production_n                        ast_Production;<commit_msg>string_2_n is now generated as just string_n<commit_after>//
//  language_parser.cpp
//  TameParse
//
//  Created by <NAME> on 19/09/2011.
//  Copyright 2011 <NAME>. All rights reserved.
//

#include <sstream>

#include "TameParse/Util/stringreader.h"
#include "TameParse/Util/utf8reader.h"
#include "TameParse/Language/language_parser.h"
#include "TameParse/Language/process.h"

#include "tameparse_language.h"

using namespace std;
using namespace util;
using namespace dfa;
using namespace compiler;
using namespace language;

/// \brief Creates a new language object
language_parser::language_parser()
: m_FileDefinition(NULL, true) {
}

typedef tameparse_language::Parser_Language_n                   Parser_Language;
typedef tameparse_language::list_of_TopLevel_Block_n            list_of_TopLevel_Block;
typedef tameparse_language::TopLevel_Block_n                    ast_TopLevel_Block;
typedef tameparse_language::Language_Block_n                    ast_Language_Block;
typedef tameparse_language::Language_Definition_n               ast_Language_Definition;
typedef tameparse_language::list_of_Language_Definition_n       list_of_Language_Definition;
typedef tameparse_language::Lexeme_Definition_n                 ast_Lexeme_Definition;
typedef tameparse_language::list_of_Lexeme_Definition_n         list_of_Lexeme_Definition;
typedef tameparse_language::list_of_Keyword_Definition_n        list_of_Keyword_Definition;
typedef tameparse_language::identifier_n                        identifier;
typedef tameparse_language::regex_n                             ast_regex;
typedef tameparse_language::string_n                            ast_string;
typedef tameparse_language::character_n                         ast_character;
typedef tameparse_language::list_of_Nonterminal_Definition_n    list_of_Nonterminal_Definition;
typedef tameparse_language::list_of__pipe__Production_n         list_of__pipe__Production;
typedef tameparse_language::list_of_Simple_Ebnf_Item_n          list_of_Simple_Ebnf_Item;
typedef tameparse_language::Nonterminal_Definition_n            ast_Nonterminal_Definition;
typedef tameparse_language::Production_n                        ast_Production;
{'fraction_non_alphanumeric': 0.060382576630560035, 'fraction_numerical': 0.006453099792578935, 'mean_word_length': 3.9713631156930127, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5985876', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1177, 'n_words': 213}