<commit_before>[1]     document           ::= prolog element Misc*

[2]     Char               ::= #x9 | #xA | #xD
                             | [#x20-#xD7FF]
                             | [#xE000-#xFFFD]
                             | [#x10000-#x10FFFF]

[3]     S                  ::= (#x20 | #x9 | #xD | #xA)+

[4]     NameStartChar      ::= ":" | [A-Z] | "_" | [a-z]
                             | [#xC0-#xD6]
                             | [#xD8-#xF6]
                             | [#xF8-#x2FF]
                             | [#x370-#x37D]
                             | [#x37F-#x1FFF]
                             | [#x200C-#x200D]
                             | [#x2070-#x218F]
                             | [#x2C00-#x2FEF]
                             | [#x3001-#xD7FF]
                             | [#xF900-#xFDCF]
                             | [#xFDF0-#xFFFD]
                             | [#x10000-#xEFFFF]

[4a]    NameChar           ::= NameStartChar
                             | "-" | "." | [0-9]
                             | #xB7
                             | [#x0300-#x036F]
                             | [#x203F-#x2040]

[5]     Name               ::= NameStartChar (NameChar)*

[6]     Names              ::= Name (#x20 Name)*

[7]     Nmtoken            ::= (NameChar)+

[8]     Nmtokens           ::= Nmtoken (#x20 Nmtoken)*

[9]     EntityValue        ::= '"' ([^%&"] | PEReference | Reference)* '"'
                             | "'" ([^%&'] | PEReference | Reference)* "'"

[10]    AttValue           ::= '"' ([^<&"] | Reference)* '"'
                             | "'" ([^<&'] | Reference)* "'"

[11]    SystemLiteral      ::= ('"' [^"]* '"') | ("'" [^']* "'")

[12]    PubidLiteral       ::= '"' PubidChar* '"' | "'" (PubidChar - "'")* "'"

[13]    PubidChar          ::= #x20 | #xD | #xA
                             | [a-zA-Z0-9]
                             | [-'()+,./:=?;!*#@$_%]

[14]    CharData           ::= [^<&]* - ([^<&]* ']]>' [^<&]*)

[15]    Comment            ::= '<!--' ((Char - '-')
                             | ('-' (Char - '-')))* '-->'

[16]    PI                 ::= '<?' PITarget (S (Char* - (Char* '?>' Char*)))? '?>'

[17]    PITarget           ::= Name - (('X' | 'x') ('M' | 'm') ('L' | 'l'))

[18]    CDSect             ::= CDStart CData CDEnd

[19]    CDStart            ::= '<![CDATA['

[20]    CData              ::= (Char* - (Char* ']]>' Char*))

[21]    CDEnd              ::= ']]>'<commit_msg>The comment grammar should be one line<commit_after>[1]     document           ::= prolog element Misc*

[2]     Char               ::= #x9 | #xA | #xD
                             | [#x20-#xD7FF]
                             | [#xE000-#xFFFD]
                             | [#x10000-#x10FFFF]

[3]     S                  ::= (#x20 | #x9 | #xD | #xA)+

[4]     NameStartChar      ::= ":" | [A-Z] | "_" | [a-z]
                             | [#xC0-#xD6]
                             | [#xD8-#xF6]
                             | [#xF8-#x2FF]
                             | [#x370-#x37D]
                             | [#x37F-#x1FFF]
                             | [#x200C-#x200D]
                             | [#x2070-#x218F]
                             | [#x2C00-#x2FEF]
                             | [#x3001-#xD7FF]
                             | [#xF900-#xFDCF]
                             | [#xFDF0-#xFFFD]
                             | [#x10000-#xEFFFF]

[4a]    NameChar           ::= NameStartChar
                             | "-" | "." | [0-9]
                             | #xB7
                             | [#x0300-#x036F]
                             | [#x203F-#x2040]

[5]     Name               ::= NameStartChar (NameChar)*

[6]     Names              ::= Name (#x20 Name)*

[7]     Nmtoken            ::= (NameChar)+

[8]     Nmtokens           ::= Nmtoken (#x20 Nmtoken)*

[9]     EntityValue        ::= '"' ([^%&"] | PEReference | Reference)* '"'
                             | "'" ([^%&'] | PEReference | Reference)* "'"

[10]    AttValue           ::= '"' ([^<&"] | Reference)* '"'
                             | "'" ([^<&'] | Reference)* "'"

[11]    SystemLiteral      ::= ('"' [^"]* '"') | ("'" [^']* "'")

[12]    PubidLiteral       ::= '"' PubidChar* '"' | "'" (PubidChar - "'")* "'"

[13]    PubidChar          ::= #x20 | #xD | #xA
                             | [a-zA-Z0-9]
                             | [-'()+,./:=?;!*#@$_%]

[14]    CharData           ::= [^<&]* - ([^<&]* ']]>' [^<&]*)

[15]    Comment            ::= '<!--' ((Char - '-') | ('-' (Char - '-')))* '-->'

[16]    PI                 ::= '<?' PITarget (S (Char* - (Char* '?>' Char*)))? '?>'

[17]    PITarget           ::= Name - (('X' | 'x') ('M' | 'm') ('L' | 'l'))

[18]    CDSect             ::= CDStart CData CDEnd

[19]    CDStart            ::= '<![CDATA['

[20]    CData              ::= (Char* - (Char* ']]>' Char*))

[21]    CDEnd              ::= ']]>'
{'fraction_non_alphanumeric': 0.22321428571428573, 'fraction_numerical': 0.049107142857142856, 'mean_word_length': 0.9030888030888031, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1252657', 'n_tokens_mistral': 1847, 'n_tokens_neox': 1667, 'n_words': 242}