<commit_before>from keras.preprocessing.text import Tokenizer, maketrans

# we do not want to filter anything in code.
def base_code_filter():
    return ""

def text_to_word_sequence(text, filters=base_code_filter(), lower=True, split=" "):
    '''prune: sequence of characters to filter out
    '''
    if lower:
        text = text.lower() #type: str
    seq = []
    curr = ""
    for c in text:
        if c.isalnum():
            curr += c
        else:
            if curr != "":
                seq.append(curr)
                curr = ""
            if not c.isspace() or c == '\n':
                seq.append(c)
    return [_f for _f in seq if _f]

# this class overrides methods to use the text_to_word_sequence function above.
class CodeTokenizer(Tokenizer):
    def fit_on_texts(self, texts):
        '''Required before using texts_to_sequences or texts_to_matrix

        # Arguments
            texts: can be a list of strings,
                or a generator of strings (for memory-efficiency)
        '''
        self.document_count = 0<commit_msg>comments<commit_after>from keras.preprocessing.text import Tokenizer, maketrans

# we do not want to filter anything in code.
def base_code_filter():
    return ""

def text_to_word_sequence(text, filters=base_code_filter(), lower=True, split=" "):
    ''' Tokenizes code. All consecutive alphanumeric characters are grouped into one token.
    Thereby trying to heuristically match identifiers.
    All other symbols are seen as one token.
    Whitespace is stripped, except the newline token.
    '''
    if lower:
        text = text.lower() #type: str
    seq = []
    curr = ""
    for c in text:
        if c.isalnum():
            curr += c
        else:
            if curr != "":
                seq.append(curr)
                curr = ""
            if not c.isspace() or c == '\n':
                seq.append(c)
    return [_f for _f in seq if _f]

# this class overrides methods to be able to use the text_to_word_sequence function above.
class CodeTokenizer(Tokenizer):
    def fit_on_texts(self, texts):
        '''Required before using texts_to_sequences or texts_to_matrix

        # Arguments
            texts: can be a list of strings,
                or a generator of strings (for memory-efficiency)
        '''
        self.document_count = 0
{'fraction_non_alphanumeric': 0.08981521272024065, 'fraction_numerical': 0.0008594757198109154, 'mean_word_length': 2.1934156378600824, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1943436', 'n_tokens_mistral': 676, 'n_tokens_neox': 658, 'n_words': 233}