<commit_before>{
pattern           = pattern_name '=' alternatives text_patterns.
pattern_name      = ident [ pattern_params ].
pattern_arguments = '(' name { ',' name } ')'.
name              = ident [ '.' ident ].

alternatives      = permutation { '|' permutation } [ conditions ].
permutation       = elements { '~' elements }.
elements          = element { element }
element           = simple | complex | optional | repeat.
simple            = regexp | word.
complex           = '(' alternatives ')'.
optional          = '[' alternatives ']'.
repeat            = '{' alternatives '}' [ '<' number [ ',' number ] '>' ].

word              = ident [ '<' attribute { ',' attribute } '>' ].
attribute         = [ [ ident ] comparison ] value { '|' value }.
comparison        = '=' | '!='.
value             = ident | regexp.
regexp            = '"' cpp_regular_expression '"'.

conditions        = '<<' condition { ',' condition } '>>'.
condition         = agreement | dictionary.
agreement         = name { agreement_type name }.
agreement_type    = '=' | '=='.
dictionary        = ident '(' arguments ')'.
arguments         = names { ',' names }
names             = ident { ident }.

text_patterns     = '=text>' text_pattern { ',' text_patterns }.
text_pattern      = text_words [ '<<' text_agreements '>>' ].
text_words        = text_word { text_word }.
text_word         = ident { ident '=' value } | '#' ident | regexp.
text_agreements   = text_agreement { ',' text_agreement }.
text_agreement    = name '~>' name.
}<commit_msg>correct lspl2 ebnf syntax description<commit_after>pattern           = pattern_name '=' alternatives text_patterns.
pattern_name      = ident [ pattern_arguments ].
pattern_arguments = '(' name { ',' name } ')'.
name              = ident [ '.' ident ].

alternatives      = permutation { '|' permutation } [ conditions ].
permutation       = elements { '~' elements }.
elements          = element { element }.
element           = simple | complex | optional | repeat.
simple            = regexp | word.
complex           = '(' alternatives ')'.
optional          = '[' alternatives ']'.
repeat            = '{' alternatives '}' [ '<' number [ ',' number ] '>' ].

word              = ident [ attributes ].
attributes        = '<' attribute { ',' attribute } '>'
attribute         = [ [ ident ] comparison ] value { '|' value }.
comparison        = '=' | '!='.
value             = ident | regexp.
regexp            = '"' cpp_regular_expression '"'.

conditions        = '<<' condition { ',' condition } '>>'.
condition         = agreement | dictionary.
agreement         = name { agreement_type name }.
agreement_type    = '=' | '=='.
dictionary        = ident '(' arguments ')'.
arguments         = names { ',' names }.
names             = ident { ident }.

text_patterns     = '=text>' text_pattern { ',' text_patterns }.
text_pattern      = text_words [ '<<' text_agreements '>>' ].
text_words        = text_word { text_word }.
text_word         = ident [ text_attributes ] | '#' ident | regexp.
text_attributes   = '<' text_attribute { ',' text_attribute } '>'
text_attribute    = ident '=' value.
text_agreements   = text_agreement { ',' text_agreement }.
text_agreement    = name '~>' name.
{'fraction_non_alphanumeric': 0.15753636645001548, 'fraction_numerical': 0.00030950170225936243, 'mean_word_length': 2.2095332671300896, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4514691', 'n_tokens_mistral': 923, 'n_tokens_neox': 895, 'n_words': 204}