<commit_before>__author__ = 'Nikita'


def is_eof(ch):
    return not ch


def is_space(ch):
    return ch == ' '


def is_tab(ch):
    return ch == '\t'


def is_new_line(ch):
    return ch == '\n'


def is_split(ch):
    return is_space(ch) or is_tab(ch) or is_new_line(ch) or is_eof(ch)


def is_comma(ch):
    return ch == ','


def is_colon(ch):
    return ch == ':'


def is_semicolon(ch):
    return ch == ';'


def is_starting_comment(char):
    return char == '{'


def is_finishing_comment(char):
    return char == '}'


def is_punctuation(ch):
    return is_comma(ch) or is_colon(ch) or is_semicolon(ch) or is_starting_comment(ch)<commit_msg>add brackets<commit_after>__author__ = 'Nikita'


def is_eof(ch):
    return not ch


def is_space(ch):
    return ch == ' '


def is_tab(ch):
    return ch == '\t'


def is_new_line(ch):
    return ch == '\n'


def is_split(ch):
    return is_space(ch) or is_tab(ch) or is_new_line(ch) or is_eof(ch)


def is_comma(ch):
    return ch == ','


def is_colon(ch):
    return ch == ':'


def is_semicolon(ch):
    return ch == ';'


def is_left_curly_bracket(char):
    return char == '{'


def is_right_curly_bracket(char):
    return char == '}'


def is_left_round_bracket(char):
    return char == '('


def is_right_round_bracket(char):
    return char == ')'


def is_left_square_bracket(char):
    return char == '['


def is_right_suqare_bracket(char):
    return char == ']'


def is_punctuation(ch):
    return is_comma(ch) or is_colon(ch) or is_semicolon(ch) or is_left_curly_bracket(ch)
{'fraction_non_alphanumeric': 0.14202711426726922, 'fraction_numerical': 0.0, 'mean_word_length': 3.5857988165680474, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3281565', 'n_tokens_mistral': 652, 'n_tokens_neox': 610, 'n_words': 138}