<commit_before>//
//  GrammarCompiler.h
//  TMHighlighter
//
//  Created by david on 1/3/15.
//  Copyright (c) 2015 maxtrixcube. All rights reserved.
//

#import <Foundation/Foundation.h>
#import "Grammar.h"

#define SHL_HAS_COMPILED_KEY @"compiled"
#define SHL_CASE_INSENSITIVE @"case_insensitive"
#define SHL_KEYWORDS_KEY @"keywords"
#define SHL_KEYWORDS_KEYWORD_KEY @"keyword"
#define SHL_KEYWORDS__BUILDTIN_KEY @"built_in"
#define SHL_KEYWORDS_LITERAL_KEY @"literal"
#define SHL_LEXEMES_KEY @"lexemes"
#define SHL_LEXEMES_RE_KEY @"lexemesRe"
#define SHL_ILLEGAL_KEY @"illegal"
#define SHL_ILLEGAL_RE_KEY @"illegalRe"
#define SHL_RELEVANCE_KEY @"relevance"
#define SHL_CONTAINS_KEY @"contains"
#define SHL_BEGIN_KEY @"begin"
#define SHL_BEGIN_RE_KEY @"beginRe"
#define SHL_BEGINKEYWORDS_KEY @"beginKeywords"
#define SHL_END_KEY @"end"
#define SHL_END_RE_KEY @"endRe"
#define SHL_TERMINATOR_END_KEY @"terminator_end"
#define SHL_ENDS_WITH_PARENT_KEY @"endsWithParent"
#define SHL_VARIANTS_KEY @"variants"
#define SHL_STARTS_KEY @"starts"
#define SHL_TERMINATORS_KEY @"terminators"

#define SHL_EXCLUDE_BEGIN_KEY @"excludeBegin"
#define SHL_EXCLUDE_END_KEY @"excludeEnd"
#define SHL_RETURN_BEGIN_KEY @"returnBegin"
#define SHL_RETURN_END_KEY @"returnEnd"
#define SHL_CLASS_NAME_KEY @"className"

@interface GrammarCompiler : NSObject
- (void)compile: (Grammar*)aGrammar;
@end
<commit_msg>add definition macro for syntax name<commit_after>//
//  GrammarCompiler.h
//  TMHighlighter
//
//  Created by david on 1/3/15.
//  Copyright (c) 2015 maxtrixcube. All rights reserved.
//

#import <Foundation/Foundation.h>
#import "Grammar.h"

#define SHL_HAS_COMPILED_KEY @"compiled"
#define SHL_CASE_INSENSITIVE @"case_insensitive"
#define SHL_KEYWORDS_KEY @"keywords"
#define SHL_KEYWORDS_KEYWORD_KEY @"keyword"
#define SHL_KEYWORDS__BUILDTIN_KEY @"built_in"
#define SHL_KEYWORDS_LITERAL_KEY @"literal"
#define SHL_LEXEMES_KEY @"lexemes"
#define SHL_LEXEMES_RE_KEY @"lexemesRe"
#define SHL_ILLEGAL_KEY @"illegal"
#define SHL_ILLEGAL_RE_KEY @"illegalRe"
#define SHL_RELEVANCE_KEY @"relevance"
#define SHL_CONTAINS_KEY @"contains"
#define SHL_BEGIN_KEY @"begin"
#define SHL_BEGIN_RE_KEY @"beginRe"
#define SHL_BEGINKEYWORDS_KEY @"beginKeywords"
#define SHL_END_KEY @"end"
#define SHL_END_RE_KEY @"endRe"
#define SHL_TERMINATOR_END_KEY @"terminator_end"
#define SHL_ENDS_WITH_PARENT_KEY @"endsWithParent"
#define SHL_VARIANTS_KEY @"variants"
#define SHL_STARTS_KEY @"starts"
#define SHL_TERMINATORS_KEY @"terminators"

#define SHL_EXCLUDE_BEGIN_KEY @"excludeBegin"
#define SHL_EXCLUDE_END_KEY @"excludeEnd"
#define SHL_RETURN_BEGIN_KEY @"returnBegin"
#define SHL_RETURN_END_KEY @"returnEnd"
#define SHL_CLASS_NAME_KEY @"className"
#define SHL_SYNTAX_NAME_KEY @"name"

@interface GrammarCompiler : NSObject
- (void)compile: (Grammar*)aGrammar;
@end

{'fraction_non_alphanumeric': 0.10927035600986958, 'fraction_numerical': 0.005639760310186817, 'mean_word_length': 8.428571428571429, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5511615', 'n_tokens_mistral': 1224, 'n_tokens_neox': 1117, 'n_words': 220}