<commit_before>use common::eof;
use nom::branch::alt;
use nom::bytes::complete::{tag, tag_no_case};
use nom::combinator::peek;
use nom::sequence::terminated;
use nom::IResult;

fn keyword_follow_char(i: &[u8]) -> IResult<&[u8], &[u8]> {
    peek(alt((
        tag(" "),
        tag("\n"),
        tag(";"),
        tag("("),
        tag(")"),
        tag("\t"),
        tag(","),
        tag("="),
        eof,
    )))(i)
}

fn keyword_a_to_c(i: &[u8]) -> IResult<&[u8], &[u8]> {
    alt((
        terminated(tag_no_case("ABORT"), keyword_follow_char),
        terminated(tag_no_case("ACTION"), keyword_follow_char),
        terminated(tag_no_case("ADD"), keyword_follow_char),<commit_msg>Add comment explaining max tuple size for alt<commit_after>use common::eof;
use nom::branch::alt;
use nom::bytes::complete::{tag, tag_no_case};
use nom::combinator::peek;
use nom::sequence::terminated;
use nom::IResult;

// NOTE: Each keyword_$start_letter_to_$end_letter function uses `alt`,
// which is implemented for tuples sizes up to 21. Because of this constraint
// on maximum tuple sizes, keywords are aggregated into groups of 20

fn keyword_follow_char(i: &[u8]) -> IResult<&[u8], &[u8]> {
    peek(alt((
        tag(" "),
        tag("\n"),
        tag(";"),
        tag("("),
        tag(")"),
        tag("\t"),
        tag(","),
        tag("="),
        eof,
    )))(i)
}

fn keyword_a_to_c(i: &[u8]) -> IResult<&[u8], &[u8]> {
    alt((
        terminated(tag_no_case("ABORT"), keyword_follow_char),
        terminated(tag_no_case("ACTION"), keyword_follow_char),
        terminated(tag_no_case("ADD"), keyword_follow_char),
{'fraction_non_alphanumeric': 0.19631901840490798, 'fraction_numerical': 0.0098159509202454, 'mean_word_length': 3.4200542005420056, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5852150', 'n_tokens_mistral': 588, 'n_tokens_neox': 580, 'n_words': 118}